From 7d5e7e389aba3017887e51bd314c1b58bdb48416 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Fri, 8 Nov 2019 14:26:49 -0700 Subject: [PATCH] update vmwarevsphere to support datastore-clusters --- drivers/vmwarevsphere/cloudinit.go | 47 +++++++----- drivers/vmwarevsphere/create.go | 67 +++++++++------- drivers/vmwarevsphere/flags.go | 14 +++- drivers/vmwarevsphere/utils.go | 119 +++++++++++++++++++++++++++++ drivers/vmwarevsphere/vm.go | 29 ++++++- drivers/vmwarevsphere/vsphere.go | 13 ++-- 6 files changed, 227 insertions(+), 62 deletions(-) diff --git a/drivers/vmwarevsphere/cloudinit.go b/drivers/vmwarevsphere/cloudinit.go index cd61df4c7f..eee9b77277 100644 --- a/drivers/vmwarevsphere/cloudinit.go +++ b/drivers/vmwarevsphere/cloudinit.go @@ -26,23 +26,25 @@ func (d *Driver) cloudInit(vm *object.VirtualMachine) error { return d.cloudInitGuestInfo(vm) } - if err := d.createCloudInitIso(); err != nil { - return err - } + if d.CloudConfig == "" { + if err := d.createCloudInitIso(); err != nil { + return err + } - dss, err := d.finder.DatastoreOrDefault(d.getCtx(), d.Datastore) - if err != nil { - return err - } + ds, err := d.getVmDatastore(vm) + if err != nil { + return err + } - err = d.uploadCloudInitIso(vm, d.datacenter, dss) - if err != nil { - return err - } + err = d.uploadCloudInitIso(vm, d.datacenter, ds) + if err != nil { + return err + } - err = d.mountCloudInitIso(vm, d.datacenter, dss) - if err != nil { - return err + err = d.mountCloudInitIso(vm, d.datacenter, ds) + if err != nil { + return err + } } return nil @@ -78,13 +80,14 @@ func (d *Driver) cloudInitGuestInfo(vm *object.VirtualMachine) error { return d.applyOpts(vm, opts) } -func (d *Driver) uploadCloudInitIso(vm *object.VirtualMachine, dc *object.Datacenter, dss *object.Datastore) error { - path, err := d.getFolder(vm) +func (d *Driver) uploadCloudInitIso(vm *object.VirtualMachine, dc *object.Datacenter, ds *object.Datastore) error { + log.Infof("Uploading cloud-init.iso") + path, err := d.getVmFolder(vm) if err != nil { return err } - dsurl, err := dss.URL(d.getCtx(), dc, filepath.Join(path, isoName)) + dsurl, err := ds.URL(d.getCtx(), dc, filepath.Join(path, isoName)) if err != nil { return err } @@ -102,19 +105,20 @@ func (d *Driver) uploadCloudInitIso(vm *object.VirtualMachine, dc *object.Datace return nil } -func (d *Driver) removeCloudInitIso(vm *object.VirtualMachine, dc *object.Datacenter, dss *object.Datastore) error { +func (d *Driver) removeCloudInitIso(vm *object.VirtualMachine, dc *object.Datacenter, ds *object.Datastore) error { + log.Infof("Removing cloud-init.iso") c, err := d.getSoapClient() if err != nil { return err } - path, err := d.getFolder(vm) + path, err := d.getVmFolder(vm) if err != nil { return err } m := object.NewFileManager(c.Client) - task, err := m.DeleteDatastoreFile(d.getCtx(), dss.Path(filepath.Join(path, isoName)), dc) + task, err := m.DeleteDatastoreFile(d.getCtx(), ds.Path(filepath.Join(path, isoName)), dc) if err != nil { return err } @@ -132,6 +136,7 @@ func (d *Driver) removeCloudInitIso(vm *object.VirtualMachine, dc *object.Datace } func (d *Driver) createCloudInitIso() error { + log.Infof("Creating cloud-init.iso") //d.CloudConfig stat'ed and loaded in flag load. sshkey, err := ioutil.ReadFile(d.publicSSHKeyPath()) if err != nil { @@ -224,7 +229,7 @@ func (d *Driver) mountCloudInitIso(vm *object.VirtualMachine, dc *object.Datacen return err } - path, err := d.getFolder(vm) + path, err := d.getVmFolder(vm) if err != nil { return err } diff --git a/drivers/vmwarevsphere/create.go b/drivers/vmwarevsphere/create.go index 3f526d3eb0..5e11c0c2b0 100644 --- a/drivers/vmwarevsphere/create.go +++ b/drivers/vmwarevsphere/create.go @@ -88,21 +88,15 @@ func (d *Driver) postCreate(vm *object.VirtualMachine) error { return d.Start() } -func (d *Driver) createManual() error { +func (d *Driver) createLegacy() error { c, err := d.getSoapClient() if err != nil { return err } - dss, err := d.finder.DatastoreOrDefault(d.getCtx(), d.Datastore) - if err != nil { - return err - } - spec := types.VirtualMachineConfigSpec{ Name: d.MachineName, GuestId: "otherLinux64Guest", - Files: &types.VirtualMachineFileInfo{VmPathName: fmt.Sprintf("[%s]", dss.Name())}, NumCPUs: int32(d.CPU), MemoryMB: int64(d.Memory), } @@ -190,6 +184,15 @@ func (d *Driver) createManual() error { } } + ds, err := d.getDatastore(&spec) + if err != nil { + return err + } + + spec.Files = &types.VirtualMachineFileInfo{ + VmPathName: fmt.Sprintf("[%s]", ds.Name()), + } + task, err := folder.CreateVM(d.getCtx(), spec, d.resourcepool, d.hostsystem) if err != nil { return err @@ -202,12 +205,12 @@ func (d *Driver) createManual() error { log.Infof("Uploading Boot2docker ISO ...") vm := object.NewVirtualMachine(c.Client, info.Result.(types.ManagedObjectReference)) - vmPath, err := d.getFolder(vm) + vmPath, err := d.getVmFolder(vm) if err != nil { return err } - dsurl, err := dss.URL(d.getCtx(), d.datacenter, filepath.Join(vmPath, isoFilename)) + dsurl, err := ds.URL(d.getCtx(), d.datacenter, filepath.Join(vmPath, isoFilename)) if err != nil { return err } @@ -222,18 +225,16 @@ func (d *Driver) createManual() error { } var add []types.BaseVirtualDevice - controller, err := devices.FindDiskController("scsi") if err != nil { return err } - disk := devices.CreateDisk(controller, dss.Reference(), - dss.Path(fmt.Sprintf("%s/%s.vmdk", d.MachineName, d.MachineName))) + disk := devices.CreateDisk(controller, ds.Reference(), + ds.Path(fmt.Sprintf("%s/%s.vmdk", vmPath, d.MachineName))) // Convert MB to KB disk.CapacityInKB = int64(d.DiskSize) * 1024 - add = append(add, disk) ide, err := devices.FindIDEController("") if err != nil { @@ -245,8 +246,7 @@ func (d *Driver) createManual() error { return err } - add = append(add, devices.InsertIso(cdrom, dss.Path(fmt.Sprintf("%s/%s", d.MachineName, isoFilename)))) - + add = append(add, devices.InsertIso(cdrom, ds.Path(fmt.Sprintf("%s/%s", vmPath, isoFilename)))) if err := vm.AddDevice(d.getCtx(), add...); err != nil { return err } @@ -273,11 +273,6 @@ func (d *Driver) createFromVmName() error { return err } - dss, err := d.finder.DatastoreOrDefault(d.getCtx(), d.Datastore) - if err != nil { - return err - } - var info *types.TaskInfo ref := d.resourcepool.Reference() spec := types.VirtualMachineCloneSpec{ @@ -286,12 +281,23 @@ func (d *Driver) createFromVmName() error { }, Config: &types.VirtualMachineConfigSpec{ GuestId: "otherLinux64Guest", - Files: &types.VirtualMachineFileInfo{VmPathName: fmt.Sprintf("[%s]", dss.Name())}, NumCPUs: int32(d.CPU), MemoryMB: int64(d.Memory), }, } + ds, err := d.getDatastore(spec.Config) + if err != nil { + return err + } + + spec.Config.Files = &types.VirtualMachineFileInfo{ + VmPathName: fmt.Sprintf("[%s]", ds.Name()), + } + + dsref := ds.Reference() + spec.Location.Datastore = &dsref + vm2Clone, err := d.fetchVM(d.CloneFrom) if err != nil { return err @@ -301,8 +307,15 @@ func (d *Driver) createFromVmName() error { if err != nil { return err } + folder := folders.VmFolder + if d.Folder != "" { + folder, err = d.finder.Folder(d.getCtx(), fmt.Sprintf("%s/%s", folders.VmFolder.InventoryPath, d.Folder)) + if err != nil { + return err + } + } - task, err := vm2Clone.Clone(d.getCtx(), folders.VmFolder, d.MachineName, spec) + task, err := vm2Clone.Clone(d.getCtx(), folder, d.MachineName, spec) if err != nil { return err } @@ -327,11 +340,6 @@ func (d *Driver) createFromLibraryName() error { return err } - ds, err := d.finder.DatastoreOrDefault(d.getCtx(), d.Datastore) - if err != nil { - return err - } - folders, err := d.datacenter.Folders(d.getCtx()) if err != nil { return err @@ -374,6 +382,11 @@ func (d *Driver) createFromLibraryName() error { hostId = d.hostsystem.Reference().Value } + ds, err := d.getDatastore(&types.VirtualMachineConfigSpec{}) + if err != nil { + return err + } + deploy := vcenter.Deploy{ DeploymentSpec: vcenter.DeploymentSpec{ Name: d.MachineName, diff --git a/drivers/vmwarevsphere/flags.go b/drivers/vmwarevsphere/flags.go index 9da3e84a98..5c28d60e2c 100644 --- a/drivers/vmwarevsphere/flags.go +++ b/drivers/vmwarevsphere/flags.go @@ -67,17 +67,22 @@ func (d *Driver) GetCreateFlags() []mcnflag.Flag { mcnflag.StringSliceFlag{ EnvVar: "VSPHERE_NETWORK", Name: "vmwarevsphere-network", - Usage: "vSphere network where the docker VM will be attached", + Usage: "vSphere network where the virtual machine will be attached", }, mcnflag.StringFlag{ EnvVar: "VSPHERE_DATASTORE", Name: "vmwarevsphere-datastore", - Usage: "vSphere datastore for docker VM", + Usage: "vSphere datastore for virtual machine", + }, + mcnflag.StringFlag{ + EnvVar: "VSPHERE_DATASTORE_CLUSTER", + Name: "vmwarevsphere-datastore-cluster", + Usage: "vSphere datastore cluster for virtual machine", }, mcnflag.StringFlag{ EnvVar: "VSPHERE_DATACENTER", Name: "vmwarevsphere-datacenter", - Usage: "vSphere datacenter for docker VM", + Usage: "vSphere datacenter for virtual machine", }, mcnflag.StringFlag{ EnvVar: "VSPHERE_FOLDER", @@ -199,6 +204,7 @@ func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error { d.Tags = flags.StringSlice("vmwarevsphere-tag") d.CustomAttributes = flags.StringSlice("vmwarevsphere-custom-attribute") d.Datastore = flags.String("vmwarevsphere-datastore") + d.DatastoreCluster = flags.String("vmwarevsphere-datastore-cluster") d.Datacenter = flags.String("vmwarevsphere-datacenter") // Sanitize input on ingress. d.Folder = strings.Trim(flags.String("vmwarevsphere-folder"), "/") @@ -234,7 +240,7 @@ func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error { if d.CreationType != "legacy" { d.CloneFrom = flags.String("vmwarevsphere-clone-from") if d.CloneFrom == "" { - return fmt.Errorf("Creation type clone needs a VM name to clone from, use --vmwarevsphere-clone-from.") + return fmt.Errorf("creation type clone needs a VM name to clone from, use --vmwarevsphere-clone-from") } } diff --git a/drivers/vmwarevsphere/utils.go b/drivers/vmwarevsphere/utils.go index f9e49a73fe..0893333672 100644 --- a/drivers/vmwarevsphere/utils.go +++ b/drivers/vmwarevsphere/utils.go @@ -10,9 +10,12 @@ import ( "github.com/docker/machine/libmachine/log" "github.com/vmware/govmomi" "github.com/vmware/govmomi/guest" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/vapi/library" "github.com/vmware/govmomi/vapi/rest" "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" "golang.org/x/net/context" ) @@ -35,6 +38,122 @@ func (d *Driver) remoteExec(procman *guest.ProcessManager, arg string) (int64, e return code, nil } +func (d *Driver) getDatastore(spec *types.VirtualMachineConfigSpec) (*object.Datastore, error) { + var sp *object.StoragePod + if d.DatastoreCluster != "" { + log.Infof("Finding datastore cluster %s", d.DatastoreCluster) + sp, err := d.finder.DatastoreCluster(d.getCtx(), d.DatastoreCluster) + if err != nil { + return nil, err + } + + return d.recommendDatastore(sp, spec) + } + + if d.Datastore != "" { + log.Infof("Finding datastore %s", d.Datastore) + return d.finder.Datastore(d.getCtx(), d.Datastore) + } + + //nothing set, try default ds cluster then default ds + log.Infof("Finding default datastore cluster") + sp, err := d.finder.DefaultDatastoreCluster(d.getCtx()) + if err != nil { + return nil, err + } + + if sp != nil { + return d.recommendDatastore(sp, spec) + } + + log.Infof("No default datastore cluster, finding default datastore") + return d.finder.DefaultDatastore(d.getCtx()) +} + +func (d *Driver) recommendDatastore(sp *object.StoragePod, spec *types.VirtualMachineConfigSpec) (*object.Datastore, error) { + // Build pod selection spec from config spec + spref := sp.Folder.Reference() + podSelectionSpec := types.StorageDrsPodSelectionSpec{ + StoragePod: &spref, + } + + // Keep list of disks that need to be placed + var disks []*types.VirtualDisk + // Collect disks eligible for placement + for _, deviceConfigSpec := range spec.DeviceChange { + s := deviceConfigSpec.GetVirtualDeviceConfigSpec() + if s.Operation != types.VirtualDeviceConfigSpecOperationAdd { + continue + } + + if s.FileOperation != types.VirtualDeviceConfigSpecFileOperationCreate { + continue + } + + d, ok := s.Device.(*types.VirtualDisk) + if !ok { + continue + } + + podConfigForPlacement := types.VmPodConfigForPlacement{ + StoragePod: spref, + Disk: []types.PodDiskLocator{ + { + DiskId: d.Key, + DiskBackingInfo: d.Backing, + }, + }, + } + + podSelectionSpec.InitialVmConfig = append(podSelectionSpec.InitialVmConfig, podConfigForPlacement) + disks = append(disks, d) + } + + rp := d.resourcepool.Reference() + sps := types.StoragePlacementSpec{ + Type: string(types.StoragePlacementSpecPlacementTypeCreate), + ResourcePool: &rp, + PodSelectionSpec: podSelectionSpec, + ConfigSpec: spec, + } + + c, err := d.getSoapClient() + if err != nil { + return nil, err + } + + srm := object.NewStorageResourceManager(c.Client) + result, err := srm.RecommendDatastores(d.getCtx(), sps) + if err != nil { + return nil, err + } + + // Use result to pin disks to recommended datastores + recs := result.Recommendations + if len(recs) == 0 { + return nil, fmt.Errorf("no datastore-cluster recommendations") + } + + ds := recs[0].Action[0].(*types.StoragePlacementAction).Destination + + var mds mo.Datastore + err = property.DefaultCollector(c.Client).RetrieveOne(d.getCtx(), ds, []string{"name"}, &mds) + if err != nil { + return nil, err + } + + datastore := object.NewDatastore(c.Client, ds) + datastore.InventoryPath = mds.Name + + // Apply recommendation to eligible disks + for _, disk := range disks { + backing := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo) + backing.Datastore = &ds + } + + return datastore, nil +} + func (d *Driver) publicSSHKeyPath() string { return d.GetSSHKeyPath() + ".pub" } diff --git a/drivers/vmwarevsphere/vm.go b/drivers/vmwarevsphere/vm.go index e370a78e4c..855e711801 100644 --- a/drivers/vmwarevsphere/vm.go +++ b/drivers/vmwarevsphere/vm.go @@ -15,7 +15,7 @@ import ( "github.com/vmware/govmomi/vim25/types" ) -func (d *Driver) getFolder(vm *object.VirtualMachine) (string, error) { +func (d *Driver) getVmFolder(vm *object.VirtualMachine) (string, error) { var mvm mo.VirtualMachine c, err := d.getSoapClient() if err != nil { @@ -34,6 +34,31 @@ func (d *Driver) getFolder(vm *object.VirtualMachine) (string, error) { return path, nil } +func (d *Driver) getVmDatastore(vm *object.VirtualMachine) (*object.Datastore, error) { + var mvm mo.VirtualMachine + c, err := d.getSoapClient() + if err != nil { + return nil, err + } + + err = c.RetrieveOne(d.getCtx(), vm.Reference(), nil, &mvm) + if err != nil { + return nil, err + } + + if len(mvm.Datastore) == 0 { + return nil, fmt.Errorf("No datastores for this VM") + } + + var ds mo.Datastore + err = c.RetrieveOne(d.getCtx(), mvm.Datastore[0], nil, &ds) + if err != nil { + return nil, err + } + + return d.finder.Datastore(d.getCtx(), ds.Name) //convert mo to object +} + func (d *Driver) fetchVM(vmname string) (*object.VirtualMachine, error) { if d.vms[vmname] != nil { return d.vms[vmname], nil @@ -98,7 +123,7 @@ func (d *Driver) addNetworks(vm *object.VirtualMachine, networks map[string]obje return err } - log.Infof("adding network: %s", netName) + log.Infof("Adding network: %s", netName) add = append(add, netdev) } diff --git a/drivers/vmwarevsphere/vsphere.go b/drivers/vmwarevsphere/vsphere.go index f61f19cd3a..4f89786c02 100644 --- a/drivers/vmwarevsphere/vsphere.go +++ b/drivers/vmwarevsphere/vsphere.go @@ -55,6 +55,7 @@ type Driver struct { Tags []string CustomAttributes []string Datastore string + DatastoreCluster string Datacenter string Folder string Pool string @@ -270,10 +271,6 @@ func (d *Driver) PreCreateCheck() error { } } - if _, err := f.DatastoreOrDefault(d.getCtx(), d.Datastore); err != nil { - return err - } - if d.CreationType == "clone" && d.ContentLibrary == "" { if _, err := d.fetchVM(d.CloneFrom); err != nil { return fmt.Errorf("Error finding vm or template to clone: %s", err) @@ -341,7 +338,7 @@ func (d *Driver) Create() error { if err := b2dutils.CopyIsoToMachineDir(d.Boot2DockerURL, d.MachineName); err != nil { return err } - return d.createManual() + return d.createLegacy() case "vm", "template", "libary": if d.ContentLibrary != "" { log.Infof("Creating VM from /%s/%s...", d.ContentLibrary, d.CloneFrom) @@ -499,7 +496,7 @@ func (d *Driver) Remove() error { } } - dss, err := d.finder.DatastoreOrDefault(d.getCtx(), d.Datastore) + ds, err := d.getVmDatastore(vm) if err != nil { return err } @@ -508,7 +505,7 @@ func (d *Driver) Remove() error { // Remove B2D Iso from VM folder if d.CreationType == "default" { m := object.NewFileManager(c.Client) - task, err = m.DeleteDatastoreFile(d.getCtx(), dss.Path(fmt.Sprintf("%s/%s", d.MachineName, isoFilename)), d.datacenter) + task, err = m.DeleteDatastoreFile(d.getCtx(), ds.Path(fmt.Sprintf("%s/%s", d.MachineName, isoFilename)), d.datacenter) if err != nil { return err } @@ -522,7 +519,7 @@ func (d *Driver) Remove() error { } if d.CloudConfig != "" { - if err = d.removeCloudInitIso(vm, d.datacenter, dss); err != nil { + if err = d.removeCloudInitIso(vm, d.datacenter, ds); err != nil { return nil } }