Skip to content

Commit

Permalink
flavor-gen bump CPI manifests and use embed instead of hardcoding
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrischdi
chrischdi committed Jan 19, 2024
1 parent e0dc2ae commit aaad7d1
Show file tree
Hide file tree
Showing 10 changed files with 1,335 additions and 1,317 deletions.
332 changes: 50 additions & 282 deletions packaging/flavorgen/cloudprovider/cloud-controller-manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ limitations under the License.
package cloudprovider

import (
"fmt"
"errors"
"regexp"
"strings"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/yaml"

"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env"
)
Expand All @@ -35,295 +35,63 @@ const (
CPIControllerImageURL = "gcr.io/cloud-provider-vsphere/cpi/release/manager"
)

// CloudControllerManagerServiceAccount returns the ServiceAccount used for the cloud-controller-manager.
func CloudControllerManagerServiceAccount() *corev1.ServiceAccount {
return &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "cloud-controller-manager",
Namespace: "kube-system",
},
TypeMeta: metav1.TypeMeta{
Kind: "ServiceAccount",
APIVersion: corev1.SchemeGroupVersion.String(),
},
// CloudControllerManagerManifests returns the yaml manifests generated via
// `helm template` from https://github.com/kubernetes/cloud-provider-vsphere/tree/master/charts/vsphere-cpi
func CloudControllerManagerManifests() (string, error) {
objectList := []string{}

imageTagRemoval := regexp.MustCompile(`(image: .*):.*`)
withoutTag := imageTagRemoval.ReplaceAll([]byte(cpiManifests), []byte(`${1}:$`+env.CPIImageKubernetesVersionVar))

// ClusterResourceSet does not support `kind: List`, we have to adjust the manifests
// to remove those lists and directly add the items instead.
for _, object := range strings.Split(string(withoutTag), "---\n") {
// Ignore empty strings
if object == "" {
continue
}
// Marshal to unstructured so we can check the type
u := &unstructured.Unstructured{}
if err := yaml.Unmarshal([]byte(object), u); err != nil {
return "", err
}
// If it is not a List we can directly add the object to the slice
if u.GetKind() != "List" {
objectList = append(objectList, object)
continue
}

// If it is a list we extract the slice at `items`
items, isSlice, err := unstructured.NestedSlice(u.UnstructuredContent(), "items")
if err != nil {
return "", err
}
if !isSlice {
return "", errors.New("expected List.items to be a slice")
}

// Loop over all items, marshal them and add it to objects.
for _, item := range items {
marshaledItem, err := yaml.Marshal(item)
if err != nil {
return "", err
}
objectList = append(objectList, string(marshaledItem))
}
}
}

// CloudControllerManagerService returns a Service for the cloud-controller-manager.
func CloudControllerManagerService() *corev1.Service {
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "cloud-controller-manager",
Namespace: "kube-system",
Labels: map[string]string{
"component": "cloud-controller-manager",
},
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeNodePort,
Ports: []corev1.ServicePort{
{
Port: 443,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromInt(43001),
},
},
Selector: map[string]string{
"component": "cloud-controller-manager",
},
},
}
return "---\n" + strings.Join(objectList, "---\n"), nil
}

// CloudControllerManagerConfigMap returns a ConfigMap containing data for the cloud config file.
func CloudControllerManagerConfigMap(cloudConfig string) *corev1.ConfigMap {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "vsphere-cloud-config",
Name: "cloud-config",
Namespace: "kube-system",
},
Data: map[string]string{
"vsphere.conf": cloudConfig,
},
}
}

// CloudControllerManagerDaemonSet returns the DaemonSet which runs the cloud-controller-manager.
func CloudControllerManagerDaemonSet(args []string) *appsv1.DaemonSet {
return &appsv1.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: "vsphere-cloud-controller-manager",
Namespace: "kube-system",
Labels: map[string]string{
"component": "cloud-controller-manager",
"tier": "control-plane",
},
},
Spec: appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"name": "vsphere-cloud-controller-manager",
},
},
UpdateStrategy: appsv1.DaemonSetUpdateStrategy{
Type: appsv1.RollingUpdateDaemonSetStrategyType,
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"name": "vsphere-cloud-controller-manager",
"component": "cloud-controller-manager",
"tier": "control-plane",
},
},
Spec: corev1.PodSpec{
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: int64ptr(1001),
},
Tolerations: []corev1.Toleration{
{
Key: "node.cloudprovider.kubernetes.io/uninitialized",
Value: "true",
Effect: corev1.TaintEffectNoSchedule,
},
{
Key: "node-role.kubernetes.io/master",
Effect: corev1.TaintEffectNoSchedule,
Operator: corev1.TolerationOpExists,
},
{
Key: "node-role.kubernetes.io/control-plane",
Effect: corev1.TaintEffectNoSchedule,
Operator: corev1.TolerationOpExists,
},
{
Key: "node.kubernetes.io/not-ready",
Effect: corev1.TaintEffectNoSchedule,
Operator: corev1.TolerationOpExists,
},
},
ServiceAccountName: "cloud-controller-manager",
PriorityClassName: "system-node-critical",
Containers: []corev1.Container{
{
Name: "vsphere-cloud-controller-manager",
Image: fmt.Sprintf("%s:%s", CPIControllerImageURL, env.CPIImageKubernetesVersionVar),
Args: args,
VolumeMounts: []corev1.VolumeMount{
{
Name: "vsphere-config-volume",
MountPath: "/etc/cloud",
ReadOnly: true,
},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
},
},
},
},
HostNetwork: true,
Volumes: []corev1.Volume{
{
Name: "vsphere-config-volume",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "vsphere-cloud-config",
},
},
},
},
},
Affinity: &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: "node-role.kubernetes.io/control-plane",
Operator: corev1.NodeSelectorOpExists,
},
},
},
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: "node-role.kubernetes.io/master",
Operator: corev1.NodeSelectorOpExists,
},
},
},
},
},
},
},
},
},
},
}
}

// CloudControllerManagerClusterRole returns the ClusterRole systemLcloud-controller-manager
// used by the cloud-controller-manager.
func CloudControllerManagerClusterRole() *rbacv1.ClusterRole {
return &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: "system:cloud-controller-manager",
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"events"},
Verbs: []string{"create", "patch", "update"},
},
{
APIGroups: []string{""},
Resources: []string{"nodes"},
Verbs: []string{"*"},
},
{
APIGroups: []string{""},
Resources: []string{"nodes/status"},
Verbs: []string{"patch"},
},
{
APIGroups: []string{""},
Resources: []string{"services"},
Verbs: []string{"list", "patch", "update", "watch"},
},
{
APIGroups: []string{""},
Resources: []string{"services/status"},
Verbs: []string{"patch"},
},
{
APIGroups: []string{""},
Resources: []string{"serviceaccounts"},
Verbs: []string{"create", "get", "list", "watch", "update"},
},
{
APIGroups: []string{""},
Resources: []string{"persistentvolumes"},
Verbs: []string{"get", "list", "watch", "update"},
},
{
APIGroups: []string{""},
Resources: []string{"endpoints"},
Verbs: []string{"create", "get", "list", "watch", "update"},
},
{
APIGroups: []string{""},
Resources: []string{"secrets"},
Verbs: []string{"get", "list", "watch"},
},
{
APIGroups: []string{"coordination.k8s.io"},
Resources: []string{"leases"},
Verbs: []string{"get", "watch", "list", "update", "create"},
},
},
}
}

// CloudControllerManagerRoleBinding binds the extension-apiserver-authentication-reader
// to the cloud-controller-manager.
func CloudControllerManagerRoleBinding() *rbacv1.RoleBinding {
return &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "servicecatalog.k8s.io:apiserver-authentication-reader",
Namespace: "kube-system",
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "Role",
Name: "extension-apiserver-authentication-reader",
},
Subjects: []rbacv1.Subject{
{
APIGroup: "",
Kind: "ServiceAccount",
Name: "cloud-controller-manager",
Namespace: "kube-system",
},
{
APIGroup: "",
Kind: "User",
Name: "cloud-controller-manager",
},
},
}
}

// CloudControllerManagerClusterRoleBinding binds the system:cloud-controller-manager
// cluster role to the cloud-controller-manager.
func CloudControllerManagerClusterRoleBinding() *rbacv1.ClusterRoleBinding {
return &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "system:cloud-controller-manager",
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "system:cloud-controller-manager",
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: "cloud-controller-manager",
Namespace: "kube-system",
},
{
Kind: "User",
Name: "cloud-controller-manager",
},
},
}
}

func int64ptr(i int) *int64 {
ptr := int64(i)
return &ptr
}
Loading

0 comments on commit aaad7d1

Please sign in to comment.