Merge pull request #471 from akutz/feature/cloud-provider-config

Centralized cloud provider configuration
kubernetes-sigs · Aug 8, 2019 · 64c4c4d · 64c4c4d
2 parents b9232ad + 62fa84c
commit 64c4c4d
Show file tree

Hide file tree

Showing 52 changed files with 3,704 additions and 1,298 deletions.
diff --git a/cmd/clusterctl/examples/vsphere/cluster.yaml.template b/cmd/clusterctl/examples/vsphere/cluster.yaml.template
@@ -18,3 +18,18 @@ spec:
       password: "${VSPHERE_PASSWORD}"
       sshAuthorizedKeys:
       - "${SSH_AUTHORIZED_KEY}"
+      cloudProviderConfiguration:
+        global:
+          secretName: "cloud-provider-vsphere-credentials"
+          secretNamespace: "kube-system"
+        virtualCenter:
+          "${VSPHERE_SERVER}":
+            datacenters: "${VSPHERE_DATACENTER}"
+        network:
+          name: "${VSPHERE_NETWORK}"
+        workspace:
+          server: "${VSPHERE_SERVER}"
+          datacenter: "${VSPHERE_DATACENTER}"
+          datastore: "${VSPHERE_DATASTORE}"
+          resourcePool: "${VSPHERE_RESOURCE_POOL}"
+          folder: "${VSPHERE_FOLDER}"
diff --git a/cmd/clusterctl/examples/vsphere/machines.yaml.template b/cmd/clusterctl/examples/vsphere/machines.yaml.template
@@ -13,9 +13,6 @@ items:
         apiVersion: vsphere.cluster.k8s.io/v1alpha1
         kind: VsphereMachineProviderSpec
         datacenter: "${VSPHERE_DATACENTER}"
-        datastore: "${VSPHERE_DATASTORE}"
-        resourcePool: "${VSPHERE_RESOURCE_POOL}"
-        folder: "${VSPHERE_FOLDER}"
         network:
           devices:
           - networkName: "${VSPHERE_NETWORK}"

diff --git a/cmd/clusterctl/examples/vsphere/machineset.yaml.template b/cmd/clusterctl/examples/vsphere/machineset.yaml.template
@@ -22,9 +22,6 @@ spec:
           apiVersion: vsphere.cluster.k8s.io/v1alpha1
           kind: VsphereMachineProviderSpec
           datacenter: "${VSPHERE_DATACENTER}"
-          datastore: "${VSPHERE_DATASTORE}"
-          resourcePool: "${VSPHERE_RESOURCE_POOL}"
-          folder: "${VSPHERE_FOLDER}"
           network:
             devices:
             - networkName: "${VSPHERE_NETWORK}"

diff --git a/config/crds/vsphere_v1alpha1_vsphereclusterproviderspec.yaml b/config/crds/vsphere_v1alpha1_vsphereclusterproviderspec.yaml
@@ -33,6 +33,122 @@ spec:
           - cert
           - key
           type: object
+        cloudProviderConfiguration:
+          description: CloudProviderConfiguration holds the cluster-wide configuration
+            for the vSphere cloud provider.
+          properties:
+            disk:
+              description: Disk is the vSphere cloud provider's disk configuration.
+              properties:
+                scsiControllerType:
+                  description: SCSIControllerType defines SCSI controller to be used.
+                  type: string
+              type: object
+            global:
+              description: Global is the vSphere cloud provider's global configuration.
+              properties:
+                apiBindPort:
+                  description: APIBindPort configures the vSphere cloud controller
+                    manager API port. Defaults to 43001.
+                  type: string
+                apiDisable:
+                  description: APIDisable disables the vSphere cloud controller manager
+                    API. Defaults to true.
+                  type: boolean
+                caFile:
+                  description: CAFile Specifies the path to a CA certificate in PEM
+                    format. If not configured, the system's CA certificates will be
+                    used.
+                  type: string
+                datacenters:
+                  description: Datacenters is a CSV string of the datacenters in which
+                    VMs are located.
+                  type: string
+                insecure:
+                  description: Insecure is a flag that disables TLS peer verification.
+                  type: boolean
+                password:
+                  description: Password is the password used to access a vSphere endpoint.
+                  type: string
+                port:
+                  description: Port is the port on which the vSphere endpoint is listening.
+                    Defaults to 443.
+                  type: string
+                roundTripperCount:
+                  description: RoundTripperCount specifies the SOAP round tripper
+                    count (retries = RoundTripper - 1)
+                  format: int32
+                  type: integer
+                secretName:
+                  description: SecretName is the name of the Kubernetes secret in
+                    which the vSphere credentials are located.
+                  type: string
+                secretNamespace:
+                  description: SecretNamespace is the namespace for SecretName.
+                  type: string
+                secretsDirectory:
+                  description: 'SecretsDirectory is a directory in which secrets may
+                    be found. This may used in the event that: 1. It is not desirable
+                    to use the K8s API to watch changes to secrets 2. The cloud controller
+                    manager is not running in a K8s environment,    such as DC/OS.
+                    For example, the container storage interface (CSI) is    container
+                    orcehstrator (CO) agnostic, and should support non-K8s COs. Defaults
+                    to /etc/cloud/credentials.'
+                  type: string
+                serviceAccount:
+                  description: ServiceAccount is the Kubernetes service account used
+                    to launch the cloud controller manager. Defaults to cloud-controller-manager.
+                  type: string
+                thumbprint:
+                  description: Thumbprint is the cryptographic thumbprint of the vSphere
+                    endpoint's certificate.
+                  type: string
+                username:
+                  description: Username is the username used to access a vSphere endpoint.
+                  type: string
+              type: object
+            labels:
+              description: Labels is the vSphere cloud provider's zone and region
+                configuration.
+              properties:
+                region:
+                  description: Region is the region in which VMs are created/located.
+                  type: string
+                zone:
+                  description: Zone is the zone in which VMs are created/located.
+                  type: string
+              type: object
+            network:
+              description: Network is the vSphere cloud provider's network configuration.
+              properties:
+                name:
+                  description: Name is the name of the network to which VMs are connected.
+                  type: string
+              type: object
+            virtualCenter:
+              description: VCenter is a list of vCenter configurations.
+              type: object
+            workspace:
+              description: Workspace is the vSphere cloud provider's workspace configuration.
+              properties:
+                datacenter:
+                  description: Datacenter is the datacenter in which VMs are created/located.
+                  type: string
+                datastore:
+                  description: Datastore is the datastore in which VMs are created/located.
+                  type: string
+                folder:
+                  description: Folder is the folder in which VMs are created/located.
+                  type: string
+                resourcePool:
+                  description: ResourcePool is the resource pool in which VMs are
+                    created/located.
+                  type: string
+                server:
+                  description: Server is the IP address or FQDN of the vSphere endpoint.
+                  type: string
+              type: object
+          type: object
         clusterConfiguration:
           description: ClusterConfiguration holds the cluster-wide information used
             during a kubeadm init call.

diff --git a/config/crds/vsphere_v1alpha1_vspheremachineproviderspec.yaml b/config/crds/vsphere_v1alpha1_vspheremachineproviderspec.yaml
@@ -23,24 +23,12 @@ spec:
           description: Datacenter is the name or inventory path of the datacenter
             where this machine's VM is created/located.
           type: string
-        datastore:
-          description: Datastore is the name or inventory path of the datastore where
-            this machine's VM is created/located. When omitted, GoVmomi's DatastoreOrDefault
-            method is used to determine the default datastore. It is recommended to
-            explicitly set this value.
-          type: string
         diskGiB:
           description: DiskGiB is the size of a virtual machine's disk, in GiB. Defaults
             to the analogue property value in the template from which this machine
             is cloned.
           format: int32
           type: integer
-        folder:
-          description: Folder is the name or inventory path of the folder where this
-            machine's VM is created/located. When omitted, GoVmomi's FolderOrDefault
-            method is used to determine the default folder. It is recommended to explicitly
-            set this value.
-          type: string
         kind:
           description: 'Kind is a string value representing the REST resource this
             object represents. Servers may infer this from the endpoint the client
@@ -342,7 +330,7 @@ spec:
           properties:
             devices:
               description: Devices is the list of network devices used by the virtual
-                machine.
+                machine. TODO(akutz) Make sure at least one network matches the             ClusterSpec.CloudProviderConfiguration.Network.Name
               items:
                 properties:
                   dhcp4:
@@ -469,12 +457,6 @@ spec:
             template from which this machine is cloned.
           format: int32
           type: integer
-        resourcePool:
-          description: ResourcePool is the name or inventory path of the resource
-            pool where this machine's VM is created/located. When omitted, GoVmomi's
-            ResourcePoolOrDefault method is used to determine the default resource
-            pool. It is recommended to explicitly set this value.
-          type: string
         template:
           description: Template is the name, inventory path, or instance UUID of the
             template used to clone new machines.

diff --git a/docs/getting_started.md b/docs/getting_started.md
@@ -2,19 +2,19 @@
 
 This is a guide on how to get started with CAPV (Cluster API Provider vSphere). To learn more about cluster API in more depth, check out the the [cluster api docs page](https://cluster-api.sigs.k8s.io/).
 
-* [Getting Started](#Getting-Started)
-  * [Bootstrapping a Management Cluster with clusterctl](#Bootstrapping-a-Management-Cluster-with-clusterctl)
-    * [Install Requirements](#Install-Requirements)
+* [Getting Started](#getting-started)
+  * [Bootstrapping a Management Cluster with clusterctl](#bootstrapping-a-management-cluster-with-clusterctl)
+    * [Install Requirements](#install-requirements)
       * [clusterctl](#clusterctl)
-      * [Docker](#Docker)
-      * [Kind](#Kind)
+      * [Docker](#docker)
+      * [Kind](#kind)
       * [kubectl](#kubectl)
-    * [vSphere Requirements](#vSphere-Requirements)
-      * [vCenter Credentials](#vCenter-Credentials)
-      * [Uploading the CAPV Machine Image](#Uploading-the-CAPV-Machine-Image)
-    * [Generating YAML for the Bootstrap Cluster](#Generating-YAML-for-the-Bootstrap-Cluster)
-    * [Using clusterctl](#Using-clusterctl)
-  * [Managing Workload Clusters using the Management Cluster](#Managing-Workload-Clusters-using-the-Management-Cluster)
+    * [vSphere Requirements](#vsphere-requirements)
+      * [vCenter Credentials](#vcenter-credentials)
+      * [Uploading the CAPV Machine Image](#uploading-the-capv-machine-image)
+    * [Generating YAML for the Bootstrap Cluster](#generating-yaml-for-the-bootstrap-cluster)
+    * [Using clusterctl](#using-clusterctl)
+  * [Managing Workload Clusters using the Management Cluster](#managing-workload-clusters-using-the-management-cluster)
 
 ## Bootstrapping a Management Cluster with clusterctl
 
@@ -171,6 +171,20 @@ spec:
       server: "<REDACTED>"
       username: "<REDACTED>"
       password: "<REDACTED>"
+      cloudProviderConfiguration:
+        global:
+          secretName: "cloud-provider-vsphere-credentials"
+          secretNamespace: "kube-system"
+        virtualCenter:
+          "<REDACTED>":
+        network:
+          name: "vm-network-1"
+        workspace:
+          server: "<REDACTED>"
+          datacenter: "SDDC-Datacenter"
+          datastore: "DefaultDatastore"
+          resourcePool: "Resources"
+          folder: "vm"
 ---
 apiVersion: cluster.k8s.io/v1alpha1
 kind: Machine
@@ -184,9 +198,6 @@ spec:
       apiVersion: vsphere.cluster.k8s.io/v1alpha1
       kind: VsphereMachineProviderSpec
       datacenter: "SDDC-Datacenter"
-      datastore: "DefaultDatastore"
-      resourcePool: "Resources"
-      folder: "vm"
       network:
         devices:
         - networkName: "vm-network-1"
@@ -225,9 +236,6 @@ spec:
           apiVersion: vsphere.cluster.k8s.io/v1alpha1
           kind: VsphereMachineProviderSpec
           datacenter: "SDDC-Datacenter"
-          datastore: "DefaultDatastore"
-          resourcePool: "Resources"
-          folder: "vm"
           network:
             devices:
             - networkName: "vm-network-1"

diff --git a/go.mod b/go.mod
@@ -25,7 +25,9 @@ require (
 	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
 	golang.org/x/oauth2 v0.0.0-20190523182746-aaccbc9213b0 // indirect
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
-	golang.org/x/tools v0.0.0-20190312170243-e65039ee4138
+	golang.org/x/tools v0.0.0-20190328211700-ab21143f2384
+	gopkg.in/gcfg.v1 v1.2.3
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 	k8s.io/api v0.0.0-20190222213804-5cb15d344471
 	k8s.io/apimachinery v0.0.0-20190703205208-4cfb76a8bf76

diff --git a/go.sum b/go.sum
@@ -274,6 +274,7 @@ github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTV
 github.com/gophercloud/gophercloud v0.0.0-20190221164956-3f3cc5a566b2/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@@ -612,8 +613,8 @@ golang.org/x/tools v0.0.0-20190102213336-ca9055ed7d04/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190104182027-498d95493402/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190111214448-fc1d57b08d7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190124215303-cc6a436ffe6b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138 h1:H3uGjxCR/6Ds0Mjgyp7LMK81+LvmbvWWEnJhzk1Pi9E=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI=
@@ -646,13 +647,17 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs=
+gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/mail.v2 v2.0.0-20180731213649-a0242b2233b4/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

diff --git a/hack/generate-yaml.sh b/hack/generate-yaml.sh
@@ -209,4 +209,4 @@ EOF
 
 # If running in Docker then ensure the contents of the OUT_DIR have the
 # the same owner as the volume mounted to the /out directory.
-[ "${DOCKER_ENABLED}" ] && chown -R "$(stat -c '%u:%g' /out)" "${OUT_DIR}"
+[ "${DOCKER_ENABLED-}" ] && chown -R "$(stat -c '%u:%g' /out)" "${OUT_DIR}"
diff --git a/hack/update-generated.sh b/hack/update-generated.sh
@@ -55,6 +55,7 @@ gen-rbac() {
 }
 
 deepcopy-v1alpha1() {
+  gen-deepcopy ./pkg/apis/vsphere/v1alpha1/cloud
   gen-deepcopy ./pkg/apis/vsphere/v1alpha1
 }
 

diff --git a/pkg/apis/vsphere/v1alpha1/cloud/doc.go b/pkg/apis/vsphere/v1alpha1/cloud/doc.go
@@ -0,0 +1,29 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package cloud contains API types for the vSphere cloud provider.
+//
+// The configuration may be unmarshalled from an INI-style configuration using
+// the "gopkg.in/gcfg.v1" package.
+//
+// The configuration may be marshalled to an INI-style configuraton using a Go
+// template.
+//
+// The "gopkg.in/go-ini/ini.v1" package was investigated, but it does not
+// support reflecting a struct with a field of type "map[string]TYPE" to INI.
+//
+// +k8s:deepcopy-gen=package
+package cloud