diff --git a/conf/README.md b/conf/README.md index 752451aef3b0..6d9be462c51d 100644 --- a/conf/README.md +++ b/conf/README.md @@ -322,6 +322,9 @@ higher priority). * `odf_provider_mode_deployment` - True if you would like to enable provider mode deployment. * `client_subcription_image` - ODF subscription image details for the storageclients. * `channel_to_client_subscription` - Channel value for the odf subscription image for storageclients. +* `custom_vpc` - Applicable only for IMB Cloud IPI deployment where we want to create custom VPC and networking + with specific Address prefixes to prevent /18 CIDR to be used. +* `ip_prefix` - Applicable only for IMB Cloud IPI deployment when custom_vpc, if not specified: 27 prefix will be used. #### UPGRADE diff --git a/conf/deployment/ibmcloud/ipi_1az_us_east1_rhcos_custom_vpc_3m_3w.yaml b/conf/deployment/ibmcloud/ipi_1az_us_east1_rhcos_custom_vpc_3m_3w.yaml new file mode 100644 index 000000000000..be16aeb06f3e --- /dev/null +++ b/conf/deployment/ibmcloud/ipi_1az_us_east1_rhcos_custom_vpc_3m_3w.yaml @@ -0,0 +1,28 @@ +--- +# This conifg is for IBM Cloud IPI deployment with custom VPC where we create +# VPC and network stuff on by own in ocs-ci in us-east (Washington DC). +DEPLOYMENT: + openshift_install_timeout: 4800 + allow_lower_instance_requirements: false +ENV_DATA: + platform: 'ibm_cloud' + deployment_type: 'ipi' + region: 'us-east' + base_domain: 'ibmcloud2.qe.rh-ocs.com' + worker_instance_type: 'bx2-16x64' + master_instance_type: 'bx2-4x16' + worker_availability_zones: + - 'us-east-1' + master_availability_zones: + - 'us-east-1' + worker_replicas: 3 + master_replicas: 3 + custom_vpc: true + +# The following values need to be set in a separate config and passed to ocs-ci in +# order to deploy OCP/OCS cluster on IBM Cloud using IPI. +# +#AUTH: +# ibmcloud: +# account_id: ACCOUNT ID PLACEHOLDER +# api_key: IBM CLOUD API KEY PLACEHOLDER diff --git a/conf/deployment/ibmcloud/ipi_1az_us_south1_rhcos_custom_vpc_3m_3w.yaml b/conf/deployment/ibmcloud/ipi_1az_us_south1_rhcos_custom_vpc_3m_3w.yaml new file mode 100644 index 000000000000..42117a0169fd --- /dev/null +++ b/conf/deployment/ibmcloud/ipi_1az_us_south1_rhcos_custom_vpc_3m_3w.yaml @@ -0,0 +1,28 @@ +--- +# This conifg is for IBM Cloud IPI deployment with custom VPC where we create +# VPC and network stuff on by own in ocs-ci in us-south (Dalas). +DEPLOYMENT: + openshift_install_timeout: 4800 + allow_lower_instance_requirements: false +ENV_DATA: + platform: 'ibm_cloud' + deployment_type: 'ipi' + region: 'us-south' + base_domain: 'ibmcloud2.qe.rh-ocs.com' + worker_instance_type: 'bx2-16x64' + master_instance_type: 'bx2-4x16' + worker_availability_zones: + - 'us-south-1' + master_availability_zones: + - 'us-south-1' + worker_replicas: 3 + master_replicas: 3 + custom_vpc: true + +# The following values need to be set in a separate config and passed to ocs-ci in +# order to deploy OCP/OCS cluster on IBM Cloud using IPI. +# +#AUTH: +# ibmcloud: +# account_id: ACCOUNT ID PLACEHOLDER +# api_key: IBM CLOUD API KEY PLACEHOLDER diff --git a/ocs_ci/deployment/ibmcloud.py b/ocs_ci/deployment/ibmcloud.py index ee9a0cd003bd..2fd1613b7ed3 100644 --- a/ocs_ci/deployment/ibmcloud.py +++ b/ocs_ci/deployment/ibmcloud.py @@ -27,6 +27,7 @@ from ocs_ci.utility.deployment import get_ocp_release_image_from_installer from ocs_ci.utility.retry import retry from ocs_ci.utility.utils import ( + get_random_str, exec_cmd, get_infra_id_from_openshift_install_state, ) @@ -173,6 +174,8 @@ def deploy_ocp(self, log_cli_level="DEBUG"): f"Switching region to {other_region} due to lack of load balancers" ) ibmcloud.set_region(other_region) + if config.ENV_DATA.get("custom_vpc"): + self.prepare_custom_vpc_and_network() self.ocp_deployment = self.OCPDeployment() self.ocp_deployment.deploy_prereq() @@ -499,3 +502,45 @@ def get_load_balancers_count(self, region=None): f"Current load balancers count in region {region} is {load_balancers_count}" ) return load_balancers_count + + def prepare_custom_vpc_and_network(self): + cluster_id = get_random_str(size=5) + config.ENV_DATA["cluster_id"] = cluster_id + cluster_name = f"{config.ENV_DATA['cluster_name']}-{cluster_id}" + resource_group = cluster_name + vpc_name = f"{cluster_name}-vpc" + worker_zones = config.ENV_DATA["worker_availability_zones"] + master_zones = config.ENV_DATA["master_availability_zones"] + ip_prefix = config.ENV_DATA.get("ip_prefix", 27) + region = config.ENV_DATA["region"] + zones = set(worker_zones + master_zones) + ip_prefixes_and_subnets = {} + ibmcloud.create_resource_group(resource_group) + ibmcloud.create_vpc(vpc_name, resource_group) + for zone in zones: + ibm_cloud_subnets = constants.IBM_CLOUD_SUBNETS[region] + ip_prefixes_and_subnets[zone] = ibmcloud.find_free_network_subnets( + ibm_cloud_subnets[zone], ip_prefix + ) + for zone, ip_prefix_and_subnets in ip_prefixes_and_subnets.items(): + gateway_name = f"{cluster_name}-public-gateway-{zone}" + address_prefix, subnet_split1, subnet_split2 = ip_prefix_and_subnets + ibmcloud.create_address_prefix( + f"{cluster_name}-{zone}", vpc_name, zone, address_prefix + ) + ibmcloud.create_public_gateway(gateway_name, vpc_name, zone, resource_group) + for subnet_type, subnet in ( + ("control-plane", subnet_split1), + ("compute", subnet_split2), + ): + subnet_type_key = f"{subnet_type.replace('-', '_')}_subnets" + if not config.ENV_DATA.get(subnet_type_key): + config.ENV_DATA[subnet_type_key] = [] + subnet_name = f"{cluster_name}-subnet-{subnet_type}-{zone}" + config.ENV_DATA[subnet_type_key].append(subnet_name) + ibmcloud.create_subnet( + subnet_name, vpc_name, zone, subnet, resource_group + ) + ibmcloud.attach_subnet_to_public_gateway( + subnet_name, gateway_name, vpc_name + ) diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 23e1f79a593d..9ba35844d2b8 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1423,6 +1423,21 @@ HCI_VSPHERE, ] +IBM_CLOUD_SUBNETS = { + # Washington D.C. + "us-east": { + "us-east-1": "10.241.0.0/18", + "us-east-2": "10.241.64.0/18", + "us-east-3": "10.241.128.0/18", + }, + # Dalas + "us-south": { + "us-south-1": "10.240.0.0/18", + "us-south-2": "10.240.64.0/18", + "us-south-3": "10.240.128.0/18", + }, +} + HCI_PC_OR_MS_PLATFORM = MANAGED_SERVICE_PLATFORMS + HCI_PROVIDER_CLIENT_PLATFORMS # AWS i3 worker instance for LSO AWS_LSO_WORKER_INSTANCE = "i3en.2xlarge" diff --git a/ocs_ci/templates/ocp-deployment/install-config-ibm_cloud-ipi.yaml.j2 b/ocs_ci/templates/ocp-deployment/install-config-ibm_cloud-ipi.yaml.j2 index 52c41bc0932d..ad2b693f72b7 100644 --- a/ocs_ci/templates/ocp-deployment/install-config-ibm_cloud-ipi.yaml.j2 +++ b/ocs_ci/templates/ocp-deployment/install-config-ibm_cloud-ipi.yaml.j2 @@ -47,5 +47,18 @@ fips: {{ fips }} platform: ibmcloud: region: {{ region | default('us-south') }} +{% if custom_vpc %} + resourceGroupName: {{ cluster_name }}-{{ cluster_id }} + networkResourceGroupName: {{ cluster_name }}-{{ cluster_id }} + vpcName: {{ cluster_name }}-{{ cluster_id }}-vpc + controlPlaneSubnets: +{% for control_plane_subnet in control_plane_subnets %} + - {{ control_plane_subnet }} +{% endfor %} + computeSubnets: +{% for compute_subnet in compute_subnets %} + - {{ compute_subnet }} +{% endfor %} +{% endif %} publish: External pullSecret: '' diff --git a/ocs_ci/utility/ibmcloud.py b/ocs_ci/utility/ibmcloud.py index 605c3e5342fe..0351b92d2030 100644 --- a/ocs_ci/utility/ibmcloud.py +++ b/ocs_ci/utility/ibmcloud.py @@ -10,6 +10,7 @@ import re import requests import time +import ipaddress from copy import copy from json import JSONDecodeError from ocs_ci.framework import config @@ -727,3 +728,148 @@ def cleanup_policies_and_service_ids(cluster_name, get_infra_id_from_metadata=Tr delete_account_policy(policy["id"], api_token) for service_id in service_ids: delete_service_id(service_id["id"]) + + +def create_resource_group(resource_group): + """ + Create resource group. + + Args: + resource_group (str): resource group name + + """ + run_ibmcloud_cmd(f"ibmcloud resource group-create {resource_group}") + + +def create_vpc(cluster_name, resource_group): + """ + Create VPC. + + Args: + cluster_name (str): cluster name + resource_group (str): resource group name + + """ + run_ibmcloud_cmd( + f"ibmcloud is vpc-create {cluster_name} --address-prefix-management manual" + f" --resource-group-name {resource_group}" + ) + + +def get_used_subnets(): + """ + Get currently used subnets in IBM Cloud + + Returns: + list: subnets + """ + subnets_data = json.loads(run_ibmcloud_cmd("ibmcloud is subnets --output json")) + return [subnet["ipv4_cidr_block"] for subnet in subnets_data] + + +def create_address_prefix(prefix_name, vpc, zone, cidr): + """ + Create address prefix in VPC. + + Args: + prefix_name (str): address prefix name to create + vpc (str): VPC name + zone (str): zone name + cidr (str): CIDR for address prefix + + """ + run_ibmcloud_cmd( + f"ibmcloud is vpc-address-prefix-create {prefix_name} {vpc} {zone} {cidr}" + ) + + +def create_subnet(subnet_name, vpc, zone, cidr, resource_group): + """ + Create address prefix in VPC. + + Args: + prefix_name (str): address prefix name to create + vpc (str): VPC name + zone (str): zone name + cidr (str): CIDR for address prefix + resource_group (str): resource group name + + """ + run_ibmcloud_cmd( + f"ibmcloud is subnet-create {subnet_name} {vpc} --zone {zone} --ipv4-cidr-block {cidr}" + f" --resource-group-name {resource_group}" + ) + + +def create_public_gateway(gateway_name, vpc, zone, resource_group): + """ + Create address prefix in VPC. + + Args: + gateway_name (str): public gateway name + vpc (str): VPC name + zone (str): zone name + resource_group (str): resource group name + + """ + run_ibmcloud_cmd( + f"ibmcloud is public-gateway-create {gateway_name} {vpc} {zone} --resource-group-name {resource_group}" + ) + + +def attach_subnet_to_public_gateway(subnet_name, gateway_name, vpc): + """ + Attach subnet to public gateway. + + Args: + subnet_name (str): subnet name to attach to public gateway + gateway_name (str): public gateway name + vpc (str): VPC name + + """ + run_ibmcloud_cmd( + f"ibmcloud is subnet-update {subnet_name} --pgw {gateway_name} --vpc {vpc}" + ) + + +def find_free_network_subnets(subnet_cidr, network_prefix=27): + """ + This function will look for currently used subnet, and will try to find one which + is not occupied by any other VPC. + + Args: + subnet_cidr (str): subnet CIDR in which range to look for free subnet (e.g. 10.240.0.0/18) + network_prefix (int): subnet prefix to look for + + Returns: + tuple: (network_with_prefix, network_split_1, network_split2), where + network_with_prefix - is network CIDR which we are looking for. + network_split1 - is first CIDR split of network_with_prefix + network_split2 - is second CIDR split of network_with_prefix + """ + network = ipaddress.ip_network(subnet_cidr) + + # Get all possible /network_prefix+1 networks within the /network_prefix network + main_subnets = list(network.subnets(new_prefix=network_prefix)) + split_subnets = list(network.subnets(new_prefix=network_prefix + 1)) + zipped_subnets = [ + (main_subnets[i], split_subnets[2 * i], split_subnets[2 * i + 1]) + for i in range(len(main_subnets)) + ] + + for possible_subnets in zipped_subnets: + is_free = True + list_of_subnets = get_used_subnets() + for subnet in list_of_subnets: + for possible_subnet in possible_subnets: + tested_network = ipaddress.ip_network(subnet) + is_subnet = possible_subnet.subnet_of(tested_network) + if is_subnet: + logger.debug( + f"Subnet {possible_subnet} is subnet of {tested_network} skipping it!" + ) + is_free = False + break + if is_free: + logger.info(f"Free set of subnets found: {possible_subnets}") + return possible_subnets