From 3a4f035462f72afec365045adb16391284bdf267 Mon Sep 17 00:00:00 2001 From: Prabhav Thali Date: Tue, 31 Dec 2024 11:08:48 +0530 Subject: [PATCH] add s390x support --- data/unpack.go | 2 +- data/vpc/config.tf | 1 + data/vpc/main.tf | 97 +++++++++++++++++++++++++++++++ data/vpc/node/main.tf | 10 ++++ data/vpc/node/outputs.tf | 6 ++ data/vpc/node/provider.tf | 8 +++ data/vpc/node/variables.tf | 3 + data/vpc/outputs.tf | 26 +++++++++ data/vpc/provider.tf | 13 +++++ data/vpc/variables.tf | 37 ++++++++++++ data/vpc/vpc/main.tf | 86 +++++++++++++++++++++++++++ data/vpc/vpc/outputs.tf | 11 ++++ data/vpc/vpc/provider.tf | 8 +++ data/vpc/vpc/variables.tf | 3 + kubetest2-tf/deployer/deployer.go | 21 +++++-- kubetest2-tf/deployer/dumplogs.go | 2 +- pkg/providers/vpc/vpc.go | 71 ++++++++++++++++++++++ pkg/tfvars/vpc/vpc.go | 13 +++++ 18 files changed, 411 insertions(+), 7 deletions(-) create mode 100644 data/vpc/config.tf create mode 100644 data/vpc/main.tf create mode 100644 data/vpc/node/main.tf create mode 100644 data/vpc/node/outputs.tf create mode 100644 data/vpc/node/provider.tf create mode 100644 data/vpc/node/variables.tf create mode 100644 data/vpc/outputs.tf create mode 100644 data/vpc/provider.tf create mode 100644 data/vpc/variables.tf create mode 100644 data/vpc/vpc/main.tf create mode 100644 data/vpc/vpc/outputs.tf create mode 100644 data/vpc/vpc/provider.tf create mode 100644 data/vpc/vpc/variables.tf create mode 100644 pkg/providers/vpc/vpc.go create mode 100644 pkg/tfvars/vpc/vpc.go diff --git a/data/unpack.go b/data/unpack.go index eed9334..7fcd13e 100644 --- a/data/unpack.go +++ b/data/unpack.go @@ -9,7 +9,7 @@ import ( ) var ( - //go:embed k8s-ansible powervs config.tf + //go:embed k8s-ansible powervs vpc config.tf dir embed.FS ) diff --git a/data/vpc/config.tf b/data/vpc/config.tf new file mode 100644 index 0000000..1830bcc --- /dev/null +++ b/data/vpc/config.tf @@ -0,0 +1 @@ +../config.tf diff --git a/data/vpc/main.tf b/data/vpc/main.tf new file mode 100644 index 0000000..2de2420 --- /dev/null +++ b/data/vpc/main.tf @@ -0,0 +1,97 @@ +data "ibm_is_vpc" "vpc" { + count = var.vpc_name == "" ? 0 : 1 + name = var.vpc_name +} + +data "ibm_is_subnet" "subnet" { + count = var.vpc_subnet_name == "" ? 0 : 1 + name = var.vpc_subnet_name +} + +data "ibm_resource_group" "default_group" { + name = var.vpc_resource_group +} + +module "vpc" { + # Create new vpc and subnet only if vpc_name is not set + count = var.vpc_name == "" ? 1 : 0 + source = "./vpc" + cluster_name = var.cluster_name + zone = var.vpc_zone + resource_group = data.ibm_resource_group.default_group.id +} + +locals { + vpc_id = var.vpc_name == "" ? module.vpc[0].vpc_id : data.ibm_is_vpc.vpc[0].id + subnet_id = var.vpc_name == "" ? module.vpc[0].subnet_id : data.ibm_is_subnet.subnet[0].id + security_group_id = var.vpc_name == "" ? module.vpc[0].security_group_id : data.ibm_is_vpc.vpc[0].default_security_group +} + +data "ibm_is_image" "node_image" { + name = var.node_image +} + +data "ibm_is_ssh_key" "ssh_key" { + name = var.vpc_ssh_key +} + +resource "ibm_is_instance_template" "node_template" { + name = "${var.cluster_name}-node-template" + image = data.ibm_is_image.node_image.id + profile = var.node_profile + vpc = local.vpc_id + zone = var.vpc_zone + resource_group = data.ibm_resource_group.default_group.id + keys = [data.ibm_is_ssh_key.ssh_key.id] + + primary_network_interface { + subnet = local.subnet_id + security_groups = [local.security_group_id] + } +} + +module "master" { + source = "./node" + node_name = "${var.cluster_name}-master" + node_instance_template_id = ibm_is_instance_template.node_template.id + resource_group = data.ibm_resource_group.default_group.id +} + +module "workers" { + source = "./node" + count = var.workers_count + node_name = "${var.cluster_name}-worker-${count.index}" + node_instance_template_id = ibm_is_instance_template.node_template.id + resource_group = data.ibm_resource_group.default_group.id +} + +resource "null_resource" "wait-for-master-completes" { + connection { + type = "ssh" + user = "root" + host = module.master.public_ip + private_key = file(var.ssh_private_key) + timeout = "20m" + } + provisioner "remote-exec" { + inline = [ + "cloud-init status -w" + ] + } +} + +resource "null_resource" "wait-for-workers-completes" { + count = var.workers_count + connection { + type = "ssh" + user = "root" + host = module.workers[count.index].public_ip + private_key = file(var.ssh_private_key) + timeout = "15m" + } + provisioner "remote-exec" { + inline = [ + "cloud-init status -w" + ] + } +} diff --git a/data/vpc/node/main.tf b/data/vpc/node/main.tf new file mode 100644 index 0000000..6cc3970 --- /dev/null +++ b/data/vpc/node/main.tf @@ -0,0 +1,10 @@ +resource "ibm_is_instance" "node" { + name = var.node_name + instance_template = var.node_instance_template_id +} + +resource "ibm_is_floating_ip" "node" { + name = "${var.node_name}-ip" + target = ibm_is_instance.node.primary_network_interface[0].id + resource_group = "${var.resource_group}" +} diff --git a/data/vpc/node/outputs.tf b/data/vpc/node/outputs.tf new file mode 100644 index 0000000..ea0d809 --- /dev/null +++ b/data/vpc/node/outputs.tf @@ -0,0 +1,6 @@ +output "public_ip" { + value = ibm_is_floating_ip.node.address +} +output "private_ip" { + value = ibm_is_instance.node.primary_network_interface.0.primary_ip.0.address +} diff --git a/data/vpc/node/provider.tf b/data/vpc/node/provider.tf new file mode 100644 index 0000000..456296e --- /dev/null +++ b/data/vpc/node/provider.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = "~> 1.50.0" + } + } +} diff --git a/data/vpc/node/variables.tf b/data/vpc/node/variables.tf new file mode 100644 index 0000000..e172e15 --- /dev/null +++ b/data/vpc/node/variables.tf @@ -0,0 +1,3 @@ +variable "node_instance_template_id" {} +variable "node_name" {} +variable "resource_group" {} diff --git a/data/vpc/outputs.tf b/data/vpc/outputs.tf new file mode 100644 index 0000000..d58e8b7 --- /dev/null +++ b/data/vpc/outputs.tf @@ -0,0 +1,26 @@ +output "vpc_id" { value = local.vpc_id } +output "ssh_key_id" { value = data.ibm_is_ssh_key.ssh_key.id } +output "subnet_id" { value = local.subnet_id } +output "security_group_id" { value = local.security_group_id } +output "region" { value = var.vpc_region } +output "zone" { value = var.vpc_zone } +output "resource_group_id" { value = data.ibm_resource_group.default_group.id } +output "masters" { + value = module.master[*].public_ip + description = "k8s master node IP addresses" +} + +output "workers" { + value = module.workers[*].public_ip + description = "k8s worker node IP addresses" +} + +output "masters_private" { + value = module.master[*].private_ip + description = "k8s master nodes private IP addresses" +} + +output "workers_private" { + value = module.workers[*].private_ip + description = "k8s worker nodes private IP addresses" +} diff --git a/data/vpc/provider.tf b/data/vpc/provider.tf new file mode 100644 index 0000000..07676e5 --- /dev/null +++ b/data/vpc/provider.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = "~> 1.50.0" + } + } +} + +provider "ibm" { + ibmcloud_api_key = var.vpc_api_key + region = var.vpc_region +} diff --git a/data/vpc/variables.tf b/data/vpc/variables.tf new file mode 100644 index 0000000..c535727 --- /dev/null +++ b/data/vpc/variables.tf @@ -0,0 +1,37 @@ +variable "vpc_api_key" { + sensitive = true +} + +variable "vpc_resource_group" { + default = "default" +} + +variable "vpc_ssh_key" {} + +variable "vpc_name" { + type = string + description = "(optional) Specify existing VPC name. If none is provided, it will create a new VPC named {cluster_name}-vpc" + default = "" +} + +variable "vpc_subnet_name" { + type = string + description = "(optional) Specify existing subnet name. If none is provided, it will create a new subnet named {cluster_name}-subnet. This must be provided if vpc_name has been set" + default = "" +} + +variable "node_image" { + default = "ibm-ubuntu-22-04-2-minimal-s390x-1" +} + +variable "node_profile" { + default = "bz2-2x8" +} + +variable "vpc_region" { + default = "eu-de" +} + +variable "vpc_zone" { + default = "eu-de-1" +} diff --git a/data/vpc/vpc/main.tf b/data/vpc/vpc/main.tf new file mode 100644 index 0000000..f20b85f --- /dev/null +++ b/data/vpc/vpc/main.tf @@ -0,0 +1,86 @@ +resource "ibm_is_vpc" "vpc" { + name = "${var.cluster_name}-vpc" + default_security_group_name = "${var.cluster_name}-security-group" + resource_group = "${var.resource_group}" +} + +resource "ibm_is_floating_ip" "gateway" { + name = "${var.cluster_name}-gateway-ip" + zone = var.zone + resource_group = "${var.resource_group}" +} + +resource "ibm_is_public_gateway" "gateway" { + name = "${var.cluster_name}-gateway" + vpc = ibm_is_vpc.vpc.id + zone = var.zone + resource_group = "${var.resource_group}" + floating_ip = { + id = ibm_is_floating_ip.gateway.id + } +} + +resource "ibm_is_subnet" "primary" { + name = "${var.cluster_name}-subnet" + vpc = ibm_is_vpc.vpc.id + zone = var.zone + resource_group = "${var.resource_group}" + total_ipv4_address_count = 256 + public_gateway = ibm_is_public_gateway.gateway.id +} + +resource "ibm_is_security_group_rule" "primary_outbound" { + group = ibm_is_vpc.vpc.default_security_group + direction = "outbound" + remote = "0.0.0.0/0" +} + +resource "ibm_is_security_group_rule" "primary_inbound" { + group = ibm_is_vpc.vpc.default_security_group + direction = "inbound" + remote = ibm_is_vpc.vpc.default_security_group +} + +resource "ibm_is_security_group_rule" "primary_ssh" { + group = ibm_is_vpc.vpc.default_security_group + direction = "inbound" + remote = "0.0.0.0/0" + + tcp { + port_min = 22 + port_max = 22 + } +} + +resource "ibm_is_security_group_rule" "primary_k8s" { + group = ibm_is_vpc.vpc.default_security_group + direction = "inbound" + remote = "0.0.0.0/0" + + tcp { + port_min = 80 + port_max = 80 + } +} + +resource "ibm_is_security_group_rule" "primary_ping" { + group = ibm_is_vpc.vpc.default_security_group + direction = "inbound" + remote = "0.0.0.0/0" + + icmp { + code = 0 + type = 8 + } +} + +resource "ibm_is_security_group_rule" "primary_api_server" { + group = ibm_is_vpc.vpc.default_security_group + direction = "inbound" + remote = "0.0.0.0/0" + + tcp { + port_min = 992 + port_max = 992 + } +} diff --git a/data/vpc/vpc/outputs.tf b/data/vpc/vpc/outputs.tf new file mode 100644 index 0000000..8885582 --- /dev/null +++ b/data/vpc/vpc/outputs.tf @@ -0,0 +1,11 @@ +output "vpc_id" { + value = ibm_is_vpc.vpc.id +} + +output "subnet_id" { + value = ibm_is_subnet.primary.id +} + +output "security_group_id" { + value = ibm_is_vpc.vpc.default_security_group +} diff --git a/data/vpc/vpc/provider.tf b/data/vpc/vpc/provider.tf new file mode 100644 index 0000000..456296e --- /dev/null +++ b/data/vpc/vpc/provider.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = "~> 1.50.0" + } + } +} diff --git a/data/vpc/vpc/variables.tf b/data/vpc/vpc/variables.tf new file mode 100644 index 0000000..09664e2 --- /dev/null +++ b/data/vpc/vpc/variables.tf @@ -0,0 +1,3 @@ +variable "cluster_name" {} +variable "zone" {} +variable "resource_group" {} diff --git a/kubetest2-tf/deployer/deployer.go b/kubetest2-tf/deployer/deployer.go index 56e1a4d..f09aad1 100644 --- a/kubetest2-tf/deployer/deployer.go +++ b/kubetest2-tf/deployer/deployer.go @@ -30,6 +30,7 @@ import ( "github.com/ppc64le-cloud/kubetest2-plugins/pkg/providers" "github.com/ppc64le-cloud/kubetest2-plugins/pkg/providers/common" "github.com/ppc64le-cloud/kubetest2-plugins/pkg/providers/powervs" + "github.com/ppc64le-cloud/kubetest2-plugins/pkg/providers/vpc" "github.com/ppc64le-cloud/kubetest2-plugins/pkg/terraform" "sigs.k8s.io/kubetest2/pkg/metadata" @@ -81,6 +82,7 @@ type deployer struct { Playbook string `desc:"name of ansible playbook to be run"` ExtraVars map[string]string `desc:"Passes extra-vars to ansible playbook, enter a string of key=value pairs"` SetKubeconfig bool `desc:"Flag to set kubeconfig"` + TargetProvider string `desc:"provider value to be used(powervs, vpc)"` } func (d *deployer) Version() string { @@ -103,7 +105,13 @@ func (d *deployer) initialize() error { if err := d.checkDependencies(); err != nil { return err } - d.provider = powervs.PowerVSProvider + + if d.TargetProvider == "vpc" { + d.provider = vpc.VPCProvider + } else { + d.provider = powervs.PowerVSProvider + } + common.CommonProvider.Initialize() d.tmpDir = common.CommonProvider.ClusterName if _, err := os.Stat(d.tmpDir); os.IsNotExist(err) { @@ -135,6 +143,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { RetryOnTfFailure: 1, Playbook: "install-k8s.yml", SetKubeconfig: true, + TargetProvider: "powervs", } flagSet, err := gpflag.Parse(d) if err != nil { @@ -150,6 +159,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { func bindFlags(d *deployer) *pflag.FlagSet { flags := pflag.NewFlagSet(Name, pflag.ContinueOnError) common.CommonProvider.BindFlags(flags) + vpc.VPCProvider.BindFlags(flags) powervs.PowerVSProvider.BindFlags(flags) return flags @@ -171,8 +181,8 @@ func (d *deployer) Up() error { } for i := 0; i <= d.RetryOnTfFailure; i++ { - path, err := terraform.Apply(d.tmpDir, "powervs", d.AutoApprove) - op, oerr := terraform.Output(d.tmpDir, "powervs") + path, err := terraform.Apply(d.tmpDir, d.TargetProvider, d.AutoApprove) + op, oerr := terraform.Output(d.tmpDir, d.TargetProvider) if err != nil { if i == d.RetryOnTfFailure { fmt.Printf("terraform.Output: %s\nterraform.Output error: %v\n", op, oerr) @@ -193,7 +203,7 @@ func (d *deployer) Up() error { inventory := AnsibleInventory{} for _, machineType := range []string{"Masters", "Workers"} { var tmp []interface{} - op, err := terraform.Output(d.tmpDir, "powervs", "-json", strings.ToLower(machineType)) + op, err := terraform.Output(d.tmpDir, d.TargetProvider, "-json", strings.ToLower(machineType)) if err != nil { return fmt.Errorf("terraform.Output failed: %v", err) @@ -310,10 +320,11 @@ func setKubeconfig(host string) error { } func (d *deployer) Down() error { + if err := d.init(); err != nil { return fmt.Errorf("down failed to init: %s", err) } - err := terraform.Destroy(d.tmpDir, "powervs", d.AutoApprove) + err := terraform.Destroy(d.tmpDir, d.TargetProvider, d.AutoApprove) if err != nil { if common.CommonProvider.IgnoreDestroy { klog.Infof("terraform.Destroy failed: %v", err) diff --git a/kubetest2-tf/deployer/dumplogs.go b/kubetest2-tf/deployer/dumplogs.go index 1839363..2569d5d 100644 --- a/kubetest2-tf/deployer/dumplogs.go +++ b/kubetest2-tf/deployer/dumplogs.go @@ -61,7 +61,7 @@ func (d *deployer) DumpClusterLogs() error { // Todo: Include provider specific logic in this section. (Includes node level information/CRI/Services, etc.) for _, machineIP := range d.machineIPs { - klog.Infof("Collecting node level information from PowerVS instance %s", machineIP) + klog.Infof("Collecting node level information from instance %s", machineIP) for logFile, command := range commandFilename { stdOut.Reset() stdErr.Reset() diff --git a/pkg/providers/vpc/vpc.go b/pkg/providers/vpc/vpc.go new file mode 100644 index 0000000..5274785 --- /dev/null +++ b/pkg/providers/vpc/vpc.go @@ -0,0 +1,71 @@ +package vpc + +import ( + "encoding/json" + "fmt" + "os" + "path" + + "github.com/ppc64le-cloud/kubetest2-plugins/pkg/providers" + "github.com/ppc64le-cloud/kubetest2-plugins/pkg/tfvars/vpc" + "github.com/spf13/pflag" +) + +const ( + Name = "vpc" +) + +var _ providers.Provider = &Provider{} + +var VPCProvider = &Provider{} + +type Provider struct { + vpc.TFVars +} + +func (p *Provider) Initialize() error { + return nil +} + +func (p *Provider) BindFlags(flags *pflag.FlagSet) { + flags.StringVar( + &p.VPCName, "vpc-name", "", "IBM Cloud VPC name", + ) + flags.StringVar( + &p.SubnetName, "vpc-subnet", "", "IBM Cloud VPC subnet", + ) + flags.StringVar( + &p.Apikey, "vpc-api-key", "", "IBM Cloud API Key used for accessing the APIs", + ) + flags.StringVar( + &p.SSHKey, "vpc-ssh-key", "", "VPC SSH Key to authenticate VSIs", + ) + flags.StringVar( + &p.Region, "vpc-region", "", "IBM Cloud VPC region name", + ) + flags.StringVar( + &p.Zone, "vpc-zone", "", "IBM Cloud VPC zone name", + ) + flags.StringVar( + &p.ResourceGroup, "vpc-resource-group", "Default", "IBM Cloud resource group name(command: ibmcloud resource groups)", + ) + flags.StringVar( + &p.NodeImageName, "vpc-node-image-name", "", "Image ID(command: ibmcloud is images)", + ) + flags.StringVar( + &p.NodeProfile, "vpc-node-profile", "", "Instance profiles to provision virtual servers(command: ibmcloud is instance-profiles)", + ) +} + +func (p *Provider) DumpConfig(dir string) error { + filename := path.Join(dir, Name+".auto.tfvars.json") + config, err := json.MarshalIndent(p.TFVars, "", " ") + if err != nil { + return fmt.Errorf("errored file converting config to json: %v", err) + } + err = os.WriteFile(filename, config, 0644) + if err != nil { + return fmt.Errorf("failed to dump the json config to: %s, err: %v", filename, err) + } + return nil +} diff --git a/pkg/tfvars/vpc/vpc.go b/pkg/tfvars/vpc/vpc.go new file mode 100644 index 0000000..68819d0 --- /dev/null +++ b/pkg/tfvars/vpc/vpc.go @@ -0,0 +1,13 @@ +package vpc + +type TFVars struct { + VPCName string `json:"vpc_name"` + SubnetName string `json:"vpc_subnet_name"` + Apikey string `json:"vpc_api_key,omitempty"` + SSHKey string `json:"vpc_ssh_key"` + Region string `json:"vpc_region"` + Zone string `json:"vpc_zone"` + ResourceGroup string `json:"vpc_resource_group"` + NodeImageName string `json:"node_image"` + NodeProfile string `json:"node_profile"` +}