diff --git a/packaging/flavorgen/flavors/files/embed.go b/packaging/flavorgen/flavors/files/embed.go new file mode 100644 index 0000000000..75eb031e99 --- /dev/null +++ b/packaging/flavorgen/flavors/files/embed.go @@ -0,0 +1,30 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package files exposes embedded files as strings. +package files + +import _ "embed" + +// KubeVIPPrepare contains the kube-vip-prepare.sh script +// +//go:embed kube-vip-prepare.sh +var KubeVipPrepare string + +// KubeVIPCleanup contains the kube-vip-cleanup.sh script +// +//go:embed kube-vip-cleanup.sh +var KubeVipCleanup string diff --git a/packaging/flavorgen/flavors/files/kube-vip-cleanup.sh b/packaging/flavorgen/flavors/files/kube-vip-cleanup.sh new file mode 100644 index 0000000000..64001bbc97 --- /dev/null +++ b/packaging/flavorgen/flavors/files/kube-vip-cleanup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Reset the workaround required for kubeadm init with kube-vip: +# xref: https://github.com/kube-vip/kube-vip/issues/684 + +sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true diff --git a/packaging/flavorgen/flavors/files/kube-vip-prepare.sh b/packaging/flavorgen/flavors/files/kube-vip-prepare.sh new file mode 100644 index 0000000000..f4d1c0142e --- /dev/null +++ b/packaging/flavorgen/flavors/files/kube-vip-prepare.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Configure the workaround required for kubeadm init with kube-vip: +# xref: https://github.com/kube-vip/kube-vip/issues/684 + +# Nothing to do for kubernetes < v1.29 +KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" +if [[ "${KUBEADM_MINOR}" -lt "29" ]]; then + return +fi + +IS_KUBEADM_INIT="false" + +# cloud-init kubeadm init +if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" +fi + +# ignition kubeadm init +if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" +fi + +if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml +fi diff --git a/packaging/flavorgen/flavors/flavors.go b/packaging/flavorgen/flavors/flavors.go index b0d67476b6..999964d12f 100644 --- a/packaging/flavorgen/flavors/flavors.go +++ b/packaging/flavorgen/flavors/flavors.go @@ -91,6 +91,17 @@ func MultiNodeTemplateWithKubeVIP() ([]runtime.Object, error) { if err != nil { return nil, err } + + // pre and post-kubeadm commands for kube-vip workaround + controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands, + "/etc/kube-vip-prepare.sh", + ) + controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands = append( + controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands, + "/etc/kube-vip-cleanup.sh", + ) + crsResourcesCPI := crs.CreateCrsResourceObjectsCPI(&clusterResourceSet) identitySecret := newIdentitySecret() @@ -158,6 +169,16 @@ func MultiNodeTemplateWithKubeVIPIgnition() ([]runtime.Object, error) { } controlPlane := newIgnitionKubeadmControlplane(machineTemplate, files) + // pre and post-kubeadm commands for kube-vip workaround + controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands, + "/etc/kube-vip-prepare.sh", + ) + controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands = append( + controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands, + "/etc/kube-vip-cleanup.sh", + ) + kubeadmJoinTemplate := newIgnitionKubeadmConfigTemplate() cluster := newCluster(vsphereCluster, &controlPlane) machineDeployment := newMachineDeployment(cluster, machineTemplate, kubeadmJoinTemplate) @@ -191,6 +212,17 @@ func MultiNodeTemplateWithKubeVIPNodeIPAM() ([]runtime.Object, error) { cpMachineTemplate := newNodeIPAMVSphereMachineTemplate(env.ClusterNameVar) workerMachineTemplate := newNodeIPAMVSphereMachineTemplate(fmt.Sprintf("%s-worker", env.ClusterNameVar)) controlPlane := newKubeadmControlplane(cpMachineTemplate, newKubeVIPFiles()) + + // pre and post-kubeadm commands for kube-vip workaround + controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands, + "/etc/kube-vip-prepare.sh", + ) + controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands = append( + controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands, + "/etc/kube-vip-cleanup.sh", + ) + kubeadmJoinTemplate := newKubeadmConfigTemplate(fmt.Sprintf("%s%s", env.ClusterNameVar, env.MachineDeploymentNameSuffix), true) cluster := newCluster(vsphereCluster, &controlPlane) machineDeployment := newMachineDeployment(cluster, workerMachineTemplate, kubeadmJoinTemplate) diff --git a/packaging/flavorgen/flavors/generators.go b/packaging/flavorgen/flavors/generators.go index f415b9a7ab..d89baf5e41 100644 --- a/packaging/flavorgen/flavors/generators.go +++ b/packaging/flavorgen/flavors/generators.go @@ -33,6 +33,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/files" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/identity" ) @@ -647,18 +648,14 @@ func kubeVIPPodSpec() *corev1.Pod { MountPath: "/etc/kubernetes/admin.conf", Name: "kubeconfig", }, + { + MountPath: "/etc/hosts", + Name: "etchosts", + }, }, }, }, HostNetwork: true, - HostAliases: []corev1.HostAlias{ - { - IP: "127.0.0.1", - Hostnames: []string{ - "kubernetes", - }, - }, - }, Volumes: []corev1.Volume{ { Name: "kubeconfig", @@ -669,6 +666,15 @@ func kubeVIPPodSpec() *corev1.Pod { }, }, }, + { + Name: "etchosts", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/kube-vip.hosts", + Type: &hostPathType, + }, + }, + }, }, }, } @@ -775,6 +781,26 @@ func newKubeVIPFiles() []bootstrapv1.File { Path: "/etc/kubernetes/manifests/kube-vip.yaml", Content: kubeVIPPod(), }, + // TODO(chrischdi) Workaround for issue X + { + Owner: "root:root", + Path: "/etc/kube-vip.hosts", + Permissions: "0644", + Content: "127.0.0.1 localhost kubernetes", + }, + + { + Owner: "root:root", + Path: "/etc/kube-vip-prepare.sh", + Permissions: "0700", + Content: files.KubeVipPrepare, + }, + { + Owner: "root:root", + Path: "/etc/kube-vip-prepare.sh", + Permissions: "0700", + Content: files.KubeVipCleanup, + }, } } diff --git a/packaging/flavorgen/flavors/patches.go b/packaging/flavorgen/flavors/patches.go index 04669d7966..78c9805142 100644 --- a/packaging/flavorgen/flavors/patches.go +++ b/packaging/flavorgen/flavors/patches.go @@ -172,7 +172,7 @@ func infraClusterPatch() clusterv1.ClusterClassPatch { } func kubeVipEnabledPatch() clusterv1.ClusterClassPatch { - return clusterv1.ClusterClassPatch{ + p := clusterv1.ClusterClassPatch{ Name: "kubeVipPodManifest", Definitions: []clusterv1.PatchDefinition{ { @@ -199,4 +199,44 @@ content: {{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipP }, }, } + + kubeVIPPatches := []clusterv1.JSONPatch{ + { + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-", + ValueFrom: &clusterv1.JSONPatchValue{Template: pointer.String("/etc/kube-vip-prepare.sh")}, + }, + { + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/-", + ValueFrom: &clusterv1.JSONPatchValue{Template: pointer.String("/etc/kube-vip-cleanup.sh")}, + }, + } + + for _, f := range newKubeVIPFiles() { + if f.Path == "/etc/kubernetes/manifests/kube-vip.yaml" { + continue + } + + kubeVIPPatches = append( + kubeVIPPatches, + clusterv1.JSONPatch{ + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files/-", + ValueFrom: &clusterv1.JSONPatchValue{ + Template: pointer.String( + fmt.Sprintf(`owner: %s +permissions: "%s" +path: "%s" +content: %s`, + f.Owner, f.Permissions, f.Path, f.Content, + )), + }, + }, + ) + } + + p.Definitions[0].JSONPatches = append(p.Definitions[0].JSONPatches, kubeVIPPatches...) + + return p } diff --git a/templates/cluster-template-ignition.yaml b/templates/cluster-template-ignition.yaml index dbf4b0ad9d..d3714268a6 100644 --- a/templates/cluster-template-ignition.yaml +++ b/templates/cluster-template-ignition.yaml @@ -123,20 +123,68 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf type: FileOrCreate name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: FileOrCreate + name: etchosts status: {} owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml permissions: "0400" + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - content: | + #!/bin/bash + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "${KUBEADM_MINOR}" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - content: | + #!/bin/bash + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" format: ignition ignition: containerLinuxConfig: @@ -210,9 +258,12 @@ spec: kubeletExtraArgs: cloud-provider: external name: $${COREOS_CUSTOM_HOSTNAME} + postKubeadmCommands: + - /etc/kube-vip-cleanup.sh preKubeadmCommands: - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml + - /etc/kube-vip-prepare.sh users: - name: core sshAuthorizedKeys: diff --git a/templates/cluster-template-node-ipam.yaml b/templates/cluster-template-node-ipam.yaml index 27e285ee83..a549cccff2 100644 --- a/templates/cluster-template-node-ipam.yaml +++ b/templates/cluster-template-node-ipam.yaml @@ -160,19 +160,67 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf type: FileOrCreate name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: FileOrCreate + name: etchosts status: {} owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - content: | + #!/bin/bash + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "${KUBEADM_MINOR}" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - content: | + #!/bin/bash + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock @@ -185,12 +233,15 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ local_hostname }}' + postKubeadmCommands: + - /etc/kube-vip-cleanup.sh preKubeadmCommands: - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - /etc/kube-vip-prepare.sh users: - name: capv sshAuthorizedKeys: diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml index c6bbc00370..b75e921b9a 100644 --- a/templates/cluster-template-topology.yaml +++ b/templates/cluster-template-topology.yaml @@ -64,16 +64,18 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf type: FileOrCreate name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: FileOrCreate + name: etchosts - name: controlPlaneIpAddr value: ${CONTROL_PLANE_ENDPOINT_IP} - name: credsSecretName diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index eca335c36b..fc01303cf5 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -150,19 +150,67 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf type: FileOrCreate name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: FileOrCreate + name: etchosts status: {} owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - content: | + #!/bin/bash + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "${KUBEADM_MINOR}" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - content: | + #!/bin/bash + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock @@ -175,12 +223,15 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ local_hostname }}' + postKubeadmCommands: + - /etc/kube-vip-cleanup.sh preKubeadmCommands: - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - /etc/kube-vip-prepare.sh users: - name: capv sshAuthorizedKeys: diff --git a/templates/clusterclass-template.yaml b/templates/clusterclass-template.yaml index b1413f9013..e1978ddad9 100644 --- a/templates/clusterclass-template.yaml +++ b/templates/clusterclass-template.yaml @@ -124,6 +124,70 @@ spec: owner: root:root path: "/etc/kubernetes/manifests/kube-vip.yaml" content: {{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipPodManifest (printf "$1 %s" .controlPlaneIpAddr)) }} + - op: add + path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/- + valueFrom: + template: /etc/kube-vip-prepare.sh + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + valueFrom: + template: /etc/kube-vip-cleanup.sh + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: |- + owner: root:root + permissions: "0644" + path: "/etc/kube-vip.hosts" + content: 127.0.0.1 localhost kubernetes + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + owner: root:root + permissions: "0700" + path: "/etc/kube-vip-prepare.sh" + content: #!/bin/bash + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "${KUBEADM_MINOR}" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + owner: root:root + permissions: "0700" + path: "/etc/kube-vip-prepare.sh" + content: #!/bin/bash + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true selector: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate