From 330c11c66afb71f906532afdde7a3a797c631004 Mon Sep 17 00:00:00 2001 From: Yassine TIJANI Date: Fri, 10 Jul 2020 20:18:04 +0200 Subject: [PATCH 1/2] add support for the kube-vip flavor Signed-off-by: Yassine TIJANI --- Makefile | 4 +- packaging/flavorgen/cmd/root.go | 2 + packaging/flavorgen/flavors/flavors.go | 20 +++- packaging/flavorgen/flavors/generators.go | 126 +++++++++++++++++++++- 4 files changed, 147 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index c1b8531faf..58dd8d0b54 100644 --- a/Makefile +++ b/Makefile @@ -269,7 +269,9 @@ manifests: $(STAGE)-version-check $(STAGE)-flavors $(MANIFEST_DIR) $(BUILD_DIR) .PHONY: flavors flavors: $(FLAVOR_DIR) - go run ./packaging/flavorgen -f multi-host > $(FLAVOR_DIR)/cluster-template.yaml + go run ./packaging/flavorgen -f multi-host > $(FLAVOR_DIR)/cluster-template-haproxy.yaml + go run ./packaging/flavorgen -f vip > $(FLAVOR_DIR)/cluster-template.yaml + .PHONY: release-flavors ## Create release flavor manifests release-flavors: release-version-check diff --git a/packaging/flavorgen/cmd/root.go b/packaging/flavorgen/cmd/root.go index 9ee5494701..7894394162 100644 --- a/packaging/flavorgen/cmd/root.go +++ b/packaging/flavorgen/cmd/root.go @@ -53,6 +53,8 @@ func RunRoot(command *cobra.Command) error { switch flavor { case "multi-host": flavors.PrintObjects(flavors.MultiNodeTemplateWithHAProxy()) + case "vip": + flavors.PrintObjects(flavors.MultiNodeTemplateWithKubeVIP()) default: return errors.Errorf("invalid flavor") } diff --git a/packaging/flavorgen/flavors/flavors.go b/packaging/flavorgen/flavors/flavors.go index dee9a6fca8..8aef958f5c 100644 --- a/packaging/flavorgen/flavors/flavors.go +++ b/packaging/flavorgen/flavors/flavors.go @@ -18,13 +18,14 @@ package flavors import ( "k8s.io/apimachinery/pkg/runtime" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3" ) func MultiNodeTemplateWithHAProxy() []runtime.Object { lb := newHAProxyLoadBalancer() vsphereCluster := newVSphereCluster(&lb) machineTemplate := newVSphereMachineTemplate() - controlPlane := newKubeadmControlplane(444, machineTemplate) + controlPlane := newKubeadmControlplane(444, machineTemplate, []bootstrapv1.File{}) kubeadmJoinTemplate := newKubeadmConfigTemplate() cluster := newCluster(vsphereCluster, &controlPlane) machineDeployment := newMachineDeployment(cluster, machineTemplate, kubeadmJoinTemplate) @@ -38,3 +39,20 @@ func MultiNodeTemplateWithHAProxy() []runtime.Object { &machineDeployment, } } + +func MultiNodeTemplateWithKubeVIP() []runtime.Object { + vsphereCluster := newVSphereCluster(nil) + machineTemplate := newVSphereMachineTemplate() + controlPlane := newKubeadmControlplane(444, machineTemplate, newKubeVIPFiles()) + kubeadmJoinTemplate := newKubeadmConfigTemplate() + cluster := newCluster(vsphereCluster, &controlPlane) + machineDeployment := newMachineDeployment(cluster, machineTemplate, kubeadmJoinTemplate) + return []runtime.Object{ + &cluster, + &vsphereCluster, + &machineTemplate, + &controlPlane, + &kubeadmJoinTemplate, + &machineDeployment, + } +} diff --git a/packaging/flavorgen/flavors/generators.go b/packaging/flavorgen/flavors/generators.go index 6c596051a9..aa10bf1bc1 100644 --- a/packaging/flavorgen/flavors/generators.go +++ b/packaging/flavorgen/flavors/generators.go @@ -18,6 +18,7 @@ package flavors import ( corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/api/v1alpha3" @@ -26,6 +27,7 @@ import ( bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3" kubeadmv1beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" + "sigs.k8s.io/yaml" ) const ( @@ -49,6 +51,8 @@ const ( vSphereSSHAuthorizedKeysVar = "${ VSPHERE_SSH_AUTHORIZED_KEY }" vSphereTemplateVar = "${ VSPHERE_TEMPLATE }" workerMachineCountVar = "${ WORKER_MACHINE_COUNT }" + controlPlaneEndpointVar = "${ CONTROL_PLANE_ENDPOINT_IP }" + vipNetworkInterfaceVar = "${ VIP_NETWORK_INTERFACE }" ) type replacement struct { @@ -156,6 +160,11 @@ func newVSphereCluster(lb *infrav1.HAProxyLoadBalancer) infrav1.VSphereCluster { Kind: lb.Kind, Name: lb.Name, } + } else { + vsphereCluster.Spec.ControlPlaneEndpoint = infrav1.APIEndpoint{ + Host: controlPlaneEndpointVar, + Port: 6443, + } } return vsphereCluster } @@ -245,7 +254,7 @@ func defaultVirtualMachineCloneSpec() infrav1.VirtualMachineCloneSpec { } } -func defaultKubeadmInitSpec() bootstrapv1.KubeadmConfigSpec { +func defaultKubeadmInitSpec(files []bootstrapv1.File) bootstrapv1.KubeadmConfigSpec { return bootstrapv1.KubeadmConfigSpec{ InitConfiguration: &kubeadmv1beta1.InitConfiguration{ NodeRegistration: defaultNodeRegistrationOptions(), @@ -262,6 +271,7 @@ func defaultKubeadmInitSpec() bootstrapv1.KubeadmConfigSpec { Users: defaultUsers(), PreKubeadmCommands: defaultPreKubeadmCommands(), UseExperimentalRetryJoin: true, + Files: files, } } @@ -331,6 +341,105 @@ func defaultPreKubeadmCommands() []string { } } +func kubeVIPPod() string { + hostPathType := v1.HostPathFileOrCreate + pod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: typeToKind(&v1.Pod{}), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-vip", + Namespace: "kube-system", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "kube-vip", + Image: "plndr/kube-vip:0.1.6", + Args: []string{ + "start", + }, + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{ + "NET_ADMIN", + "SYS_TIME", + }, + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "/etc/kubernetes/admin.conf", + Name: "kubeconfig", + }, + }, + Env: []v1.EnvVar{ + { + Name: "vip_arp", + Value: "true", + }, + { + Name: "vip_leaderelection", + Value: "true", + }, + { + Name: "vip_address", + Value: controlPlaneEndpointVar, + }, + { + Name: "lb_backendport", + Value: "6443", + }, + { + Name: "vip_addpeerstolb", + Value: "true", + }, + { + Name: "lb_name", + Value: "kcpEdnpoint", + }, + { + Name: "lb_bindtovip", + Value: "true", + }, + { + Name: "vip_interface", + Value: vipNetworkInterfaceVar, + }, + { + Name: "lb_type", + Value: "tcp", + }, + { + Name: "lb_type", + Value: "tcp", + }, + }, + }, + }, + HostNetwork: true, + Volumes: []v1.Volume{ + { + Name: "kubeconfig", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/admin.conf", + Type: &hostPathType, + }, + }, + }, + }, + }, + } + podBytes, err := yaml.Marshal(pod) + if err != nil { + panic(err) + } + return string(podBytes) +} + func newMachineDeployment(cluster clusterv1.Cluster, machineTemplate infrav1.VSphereMachineTemplate, bootstrapTemplate bootstrapv1.KubeadmConfigTemplate) clusterv1.MachineDeployment { return clusterv1.MachineDeployment{ TypeMeta: metav1.TypeMeta{ @@ -395,7 +504,18 @@ func newHAProxyLoadBalancer() infrav1.HAProxyLoadBalancer { } } -func newKubeadmControlplane(replicas int, infraTemplate infrav1.VSphereMachineTemplate) controlplanev1.KubeadmControlPlane { +func newKubeVIPFiles() []bootstrapv1.File { + return []bootstrapv1.File{ + { + Owner: "root:root", + Path: "/etc/kubernetes/manifests/kube-vip.yaml", + Content: kubeVIPPod(), + }, + } + +} + +func newKubeadmControlplane(replicas int, infraTemplate infrav1.VSphereMachineTemplate, files []bootstrapv1.File) controlplanev1.KubeadmControlPlane { return controlplanev1.KubeadmControlPlane{ TypeMeta: metav1.TypeMeta{ APIVersion: controlplanev1.GroupVersion.String(), @@ -413,7 +533,7 @@ func newKubeadmControlplane(replicas int, infraTemplate infrav1.VSphereMachineTe Kind: infraTemplate.Kind, Name: infraTemplate.Name, }, - KubeadmConfigSpec: defaultKubeadmInitSpec(), + KubeadmConfigSpec: defaultKubeadmInitSpec(files), }, } } From 9d0302155d8c7dab9f866a140ec5169d2356bf07 Mon Sep 17 00:00:00 2001 From: Yassine TIJANI Date: Fri, 24 Jul 2020 19:43:36 +0200 Subject: [PATCH 2/2] keep testing on the HAProxy flavour until we figure infrastructure for VIP testing Signed-off-by: Yassine TIJANI --- Makefile | 2 +- packaging/flavorgen/flavors/generators.go | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 58dd8d0b54..4c6705c28a 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ e2e: e2e-image e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests $(MAKE) release-manifests @mkdir -p $(E2E_TEMPLATE_DIR) - cp $(RELEASE_DIR)/cluster-template.yaml $(E2E_TEMPLATE_DIR) + cp $(RELEASE_DIR)/cluster-template-haproxy.yaml $(E2E_TEMPLATE_DIR) @echo PATH=$(PATH) @echo @echo Contents of $(TOOLS_BIN_DIR): diff --git a/packaging/flavorgen/flavors/generators.go b/packaging/flavorgen/flavors/generators.go index aa10bf1bc1..70f18b75c6 100644 --- a/packaging/flavorgen/flavors/generators.go +++ b/packaging/flavorgen/flavors/generators.go @@ -398,7 +398,7 @@ func kubeVIPPod() string { }, { Name: "lb_name", - Value: "kcpEdnpoint", + Value: "kcpEndpoint", }, { Name: "lb_bindtovip", @@ -412,10 +412,6 @@ func kubeVIPPod() string { Name: "lb_type", Value: "tcp", }, - { - Name: "lb_type", - Value: "tcp", - }, }, }, },