diff --git a/src/TopoMojo.Abstractions/Interfaces/ITopoMojoClient.cs b/src/TopoMojo.Abstractions/Interfaces/ITopoMojoClient.cs index ac03b44e..2f12c981 100644 --- a/src/TopoMojo.Abstractions/Interfaces/ITopoMojoClient.cs +++ b/src/TopoMojo.Abstractions/Interfaces/ITopoMojoClient.cs @@ -1,5 +1,5 @@ -// Copyright 2020 Carnegie Mellon University. -// Released under a MIT (SEI) license. See LICENSE.md in the project root. +// Copyright 2020 Carnegie Mellon University. +// Released under a MIT (SEI) license. See LICENSE.md in the project root. using System; using System.Threading.Tasks; @@ -9,14 +9,13 @@ namespace TopoMojo.Abstractions { public interface ITopoMojoClient { - [Obsolete] - Task Start(string problemId, GamespaceSpec workspace); Task Start(GamespaceSpec workspace); Task Stop(string problemId); Task Ticket(string vmId); Task ChangeVm(VmAction vmAction); Task BuildIso(IsoBuildSpec spec); Task Templates(int id); + Task List(Search search); } } diff --git a/src/TopoMojo.Abstractions/Models/GamespaceSpec.cs b/src/TopoMojo.Abstractions/Models/GamespaceSpec.cs index 73e4dbf1..c81b3d0b 100644 --- a/src/TopoMojo.Abstractions/Models/GamespaceSpec.cs +++ b/src/TopoMojo.Abstractions/Models/GamespaceSpec.cs @@ -1,5 +1,7 @@ -// Copyright 2020 Carnegie Mellon University. -// Released under a MIT (SEI) license. See LICENSE.md in the project root. +// Copyright 2020 Carnegie Mellon University. +// Released under a MIT (SEI) license. See LICENSE.md in the project root. + +using System; namespace TopoMojo.Models { @@ -12,7 +14,9 @@ public class NewGamespace public class GamespaceSpec { public string IsolationId { get; set; } + [Obsolete] public int WorkspaceId { get; set; } + public string WorkspaceGuid { get; set; } public VmSpec[] Vms { get; set; } = new VmSpec[] {}; public bool CustomizeTemplates { get; set; } public string Templates { get; set; } diff --git a/src/TopoMojo.Abstractions/Models/HypervisorServiceConfiguration.cs b/src/TopoMojo.Abstractions/Models/HypervisorServiceConfiguration.cs index c0c447b6..1b49a4a6 100644 --- a/src/TopoMojo.Abstractions/Models/HypervisorServiceConfiguration.cs +++ b/src/TopoMojo.Abstractions/Models/HypervisorServiceConfiguration.cs @@ -17,12 +17,21 @@ public class HypervisorServiceConfiguration { public string VmStore { get; set; } = "[topomojo] _run/"; public string DiskStore { get; set; } = "[topomojo]"; public string IsoStore { get; set; } = "[topomojo] iso/"; - public string ConsoleUrl { get; set; } public string TicketUrlHandler { get; set; } = "querystring"; //"local-app", "external-domain", "host-map", "none" public Dictionary TicketUrlHostMap { get; set; } = new Dictionary(); public VlanConfiguration Vlan { get; set; } = new VlanConfiguration(); public int KeepAliveMinutes { get; set; } = 10; public string ExcludeNetworkMask { get; set; } = "topomojo"; + public SddcConfiguration Sddc { get; set; } = new SddcConfiguration(); + } + + public class SddcConfiguration + { + public string Url { get; set; } + public string AuthUrl { get; set; } + public string OrgId { get; set; } + public string SddcId { get; set; } + public string ApiKey { get; set; } } public class VlanConfiguration diff --git a/src/TopoMojo.Abstractions/Models/VmTemplate.cs b/src/TopoMojo.Abstractions/Models/VmTemplate.cs index 6ec9504e..1493a74a 100644 --- a/src/TopoMojo.Abstractions/Models/VmTemplate.cs +++ b/src/TopoMojo.Abstractions/Models/VmTemplate.cs @@ -32,6 +32,7 @@ public class VmNet { public int Id { get; set; } public string Net { get; set; } + public string Key { get; set; } public string Type { get; set; } public string Mac { get; set; } public string Ip { get; set; } diff --git a/src/TopoMojo.Abstractions/TopoMojo.Abstractions.csproj b/src/TopoMojo.Abstractions/TopoMojo.Abstractions.csproj index 24c66005..8e52c5e7 100755 --- a/src/TopoMojo.Abstractions/TopoMojo.Abstractions.csproj +++ b/src/TopoMojo.Abstractions/TopoMojo.Abstractions.csproj @@ -8,7 +8,7 @@ TopoMojo.Abstractions MIT https://github.com/cmu-sei/TopoMojo - 1.3.1 + 1.3.3 diff --git a/src/TopoMojo.Client/TopoMojo.Client.csproj b/src/TopoMojo.Client/TopoMojo.Client.csproj index 1eb60e19..6314c1ff 100644 --- a/src/TopoMojo.Client/TopoMojo.Client.csproj +++ b/src/TopoMojo.Client/TopoMojo.Client.csproj @@ -8,7 +8,7 @@ TopoMojo.Client MIT https://github.com/cmu-sei/TopoMojo - 1.3.1 + 1.3.3 diff --git a/src/TopoMojo.Client/TopoMojoClient.cs b/src/TopoMojo.Client/TopoMojoClient.cs index c552567e..2882019e 100644 --- a/src/TopoMojo.Client/TopoMojoClient.cs +++ b/src/TopoMojo.Client/TopoMojoClient.cs @@ -55,13 +55,9 @@ public async Task Start(GamespaceSpec spec) { try { - string mdText = "> Gamespace Resources: " + String.Join(" | ", game.Vms.Select(v => $"[{v.Name}](/console/{v.Id}/{v.Name}/{spec.IsolationId})")); - data = await Client.GetStringAsync(game.WorkspaceDocument); - mdText += "\n\n" + data; - - game.Markdown = mdText; + game.Markdown = data; } catch { @@ -72,48 +68,6 @@ public async Task Start(GamespaceSpec spec) return game; } - [Obsolete] - public async Task Start(string isolationTag, GamespaceSpec spec) - { - string mdText = ""; - - var model = new NewGamespace - { - Id = isolationTag, - Workspace = spec - }; - - var result = await Client.PostAsync("gamespace", Json(model)); - - if (result.IsSuccessStatusCode) - { - - string data = await result.Content.ReadAsStringAsync(); - - var game = JsonConvert.DeserializeObject(data); - - mdText = "> Gamespace Resources: " + String.Join(" | ", game.Vms.Select(v => $"[{v.Name}](/console/{v.Id}/{v.Name}/{isolationTag})")); - - if (spec.AppendMarkdown) - { - try - { - data = await Client.GetStringAsync(game.WorkspaceDocument); - - mdText += "\n\n" + data; - } - catch - { - - } - } - - } - - return mdText; - - } - public async Task Stop(string problemId) { await Client.DeleteAsync($"gamespace/{problemId}"); diff --git a/src/TopoMojo.Core/Services/EngineService.cs b/src/TopoMojo.Core/Services/EngineService.cs index c95e2577..f643279c 100644 --- a/src/TopoMojo.Core/Services/EngineService.cs +++ b/src/TopoMojo.Core/Services/EngineService.cs @@ -55,11 +55,13 @@ public async Task Launch(GamespaceSpec spec) if (game == null) { - var workspace = await _workspaceStore.Load(spec.WorkspaceId); + var workspace = string.IsNullOrEmpty(spec.WorkspaceGuid) + ? await _workspaceStore.Load(spec.WorkspaceId) + : await _workspaceStore.Load(spec.WorkspaceGuid); if (workspace == null || !workspace.HasScope(Client.Scope)) { - _logger.LogInformation($"No audience match for workspace {spec?.WorkspaceId}: [{workspace?.Audience}] [{Client?.Scope}]"); + _logger.LogInformation($"No audience match for workspace {spec?.WorkspaceGuid}: [{workspace?.Audience}] [{Client?.Scope}]"); throw new InvalidOperationException(); } @@ -189,7 +191,7 @@ private void ApplyIso(ICollection templates, GamespaceSpec sp if (vmspec != null && vmspec.SkipIso) continue; - template.Iso = $"{_pod.Options.IsoStore}/{_options.GameEngineIsoFolder}/{spec.Iso}"; + template.Iso = $"{_options.GameEngineIsoFolder}/{spec.Iso}"; } } diff --git a/src/TopoMojo.Web/Controllers/VmController.cs b/src/TopoMojo.Web/Controllers/VmController.cs index 7fc7a875..2ade472a 100644 --- a/src/TopoMojo.Web/Controllers/VmController.cs +++ b/src/TopoMojo.Web/Controllers/VmController.cs @@ -248,11 +248,11 @@ public async Task> Ticket(string id) { case "querystring": qs = $"?vmhost={src.Host}"; - target = _pod.Options.ConsoleUrl; + target = _options.ConsoleHost; break; case "local-app": - target = $"{Request.Host.Value}/{internalHost}"; + target = $"{Request.Host.Value}{Request.PathBase}/{internalHost}"; break; case "external-domain": diff --git a/src/TopoMojo.Web/Infrastructure/HeaderInspectionMiddleware.cs b/src/TopoMojo.Web/Infrastructure/HeaderInspectionMiddleware.cs index a6ffb617..9d38e831 100644 --- a/src/TopoMojo.Web/Infrastructure/HeaderInspectionMiddleware.cs +++ b/src/TopoMojo.Web/Infrastructure/HeaderInspectionMiddleware.cs @@ -24,7 +24,7 @@ ILogger logger public async Task Invoke(HttpContext context) { - var sb = new StringBuilder($"Request Headers: {context.Request.Scheme}://{context.Request.Host} from {context.Connection.RemoteIpAddress}\n"); + var sb = new StringBuilder($"Request Headers: {context.Request.Scheme}://{context.Request.Host}{context.Request.PathBase} from {context.Connection.RemoteIpAddress}\n"); sb.AppendLine($"\t{context.Request.Method} {context.Request.Path.Value} {context.Request.Protocol}"); diff --git a/src/TopoMojo.Web/Infrastructure/JsonExceptionMiddleware.cs b/src/TopoMojo.Web/Infrastructure/JsonExceptionMiddleware.cs index 121ee72b..b604b61f 100644 --- a/src/TopoMojo.Web/Infrastructure/JsonExceptionMiddleware.cs +++ b/src/TopoMojo.Web/Infrastructure/JsonExceptionMiddleware.cs @@ -6,6 +6,7 @@ using System.Text.Json; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; using TopoMojo.Web; namespace TopoMojo.Web @@ -13,12 +14,15 @@ namespace TopoMojo.Web public class JsonExceptionMiddleware { public JsonExceptionMiddleware( - RequestDelegate next + RequestDelegate next, + ILogger logger ) { _next = next; + _logger = logger; } private readonly RequestDelegate _next; + private readonly ILogger _logger; public async Task Invoke(HttpContext context) { @@ -27,6 +31,8 @@ public async Task Invoke(HttpContext context) } catch (Exception ex) { + _logger.LogError(ex, "Error"); + if (!context.Response.HasStarted) { context.Response.StatusCode = 500; diff --git a/src/TopoMojo.Web/Options/BrandingOptions.cs b/src/TopoMojo.Web/Options/BrandingOptions.cs index 61618028..5654a1c9 100644 --- a/src/TopoMojo.Web/Options/BrandingOptions.cs +++ b/src/TopoMojo.Web/Options/BrandingOptions.cs @@ -18,6 +18,6 @@ public class BrandingOptions }; public bool IncludeSwagger { get; set; } = true; - + public string PathBase { get; set; } } } diff --git a/src/TopoMojo.Web/Startup.cs b/src/TopoMojo.Web/Startup.cs index b55e6fc4..ae5809a7 100644 --- a/src/TopoMojo.Web/Startup.cs +++ b/src/TopoMojo.Web/Startup.cs @@ -186,6 +186,9 @@ public void Configure(IApplicationBuilder app) { app.UseJsonExceptions(); + if (!string.IsNullOrEmpty(Branding.PathBase)) + app.UsePathBase(Branding.PathBase); + if (Headers.LogHeaders) app.UseHeaderInspection(); diff --git a/src/TopoMojo.Web/appsettings.conf b/src/TopoMojo.Web/appsettings.conf index 841145b1..9e99f692 100644 --- a/src/TopoMojo.Web/appsettings.conf +++ b/src/TopoMojo.Web/appsettings.conf @@ -24,10 +24,12 @@ # Branding__Title = TopoMojo # Branding__LogoUrl = +## If hosting in virtual directory, specify path base +# Branding__PathBase = + ## Disable the Swagger OpenApi host by setting to false # Branding__IncludeSwagger = true - #################### ## Caching #################### @@ -110,12 +112,9 @@ Authorization__SwaggerClient__ClientId = topomojo-swagger ## datastore path of workspace folders and template disks # Pod__DiskStore = -## Url for console connections -# Pod__ConsoleUrl = - ## TicketUrlHandler: console url transform method ## "none" : wss://./ticket/123455 -## "querystring" : wss:///ticket/123455?vmhost= +## "querystring" : wss:///ticket/123455?vmhost= # Pod__TicketUrlHandler = querystring ## range of vlan id's available to topomojo; i.e. [200-399] diff --git a/src/TopoMojo.vSphere/DatastorePath.cs b/src/TopoMojo.vSphere/DatastorePath.cs index b74d6a57..3b80c271 100644 --- a/src/TopoMojo.vSphere/DatastorePath.cs +++ b/src/TopoMojo.vSphere/DatastorePath.cs @@ -2,6 +2,8 @@ // Released under a 3 Clause BSD-style license. See LICENSE.md in the project root for license information. using System; +using System.Collections.Generic; +using System.Linq; namespace TopoMojo.vSphere { @@ -9,27 +11,7 @@ public class DatastorePath { public DatastorePath(string path) { - if (path.HasValue()) - { - _folder = path.Replace("\\", "/"); - int x = _folder.IndexOf("["); - int y = _folder.IndexOf("]"); - if (x >= 0 && y > x) - { - _ds = _folder.Substring(x+1, y-x-1); - _folder = _folder.Substring(y+1).Trim(); - } - x = _folder.LastIndexOf('/'); - _file = _folder.Substring(x+1); - if (x >= 0) - { - _folder = _folder.Substring(0, x); - } - else - { - _folder = ""; - } - } + Merge(path); } private string _ds; @@ -52,6 +34,17 @@ public string Folder set { _folder = value; } } + public string TopLevelFolder + { + get { return _folder.Split('/').First(); } + set { + var list = new List(); + list.Add(value); + list.AddRange(_folder.Split('/').Skip(1)); + _folder = string.Join("/", list); + } + } + private string _file; public string File { @@ -59,39 +52,56 @@ public string File set { _file = value;} } + /// + /// Merge an "concrete" datastore root with an "abstract" datastore root + /// + /// + /// Templates have abstract paths: [ds] folder/disk.vmdk + /// Here we merge with a concrete path: [actual] root/ + /// Result: [actual] root/folder/disk.vmdk + /// + /// public void Merge(string path) { - if (path.HasValue()) + if (!path.HasValue()) + return; + + string file = "", ds = "", folder = ""; + + folder = path.Replace("\\", "/"); + + int x = folder.IndexOf("["); + int y = folder.IndexOf("]"); + + if (x >= 0 && y > x) { - string file = "", ds = "", folder = ""; - folder = path.Replace("\\", "/"); - int x = folder.IndexOf("["); - int y = folder.IndexOf("]"); - if (x >= 0 && y > x) - { - ds = folder.Substring(x+1, y-x-1); - folder = folder.Substring(y+1).Trim(); - } - x = folder.LastIndexOf('/'); - file = folder.Substring(x+1); - if (x >= 0) - { - folder = folder.Substring(0, x); - } - else - { - folder = ""; - } - Datastore = ds; - if (Folder.HasValue() && folder.HasValue()) - folder += "/"; - Folder = folder + Folder; + ds = folder.Substring(x+1, y-x-1); + folder = folder.Substring(y+1).Trim(); } + + x = folder.LastIndexOf('/'); + + file = folder.Substring(x+1); + + folder = x >= 0 + ? folder.Substring(0, x) + : ""; + + if (Folder.HasValue() && folder.HasValue()) + folder += "/"; + + if (!_file.HasValue()) + _file = file; + + Folder = folder + Folder; + + Datastore = ds; } public override string ToString() { string separator = FolderPath.EndsWith("]") ? " " : "/"; + return String.Format("{0}{1}{2}", FolderPath, separator, File); } } diff --git a/src/TopoMojo.vSphere/DistributedNetworkManager.cs b/src/TopoMojo.vSphere/DistributedNetworkManager.cs index af2bd761..acb76a33 100644 --- a/src/TopoMojo.vSphere/DistributedNetworkManager.cs +++ b/src/TopoMojo.vSphere/DistributedNetworkManager.cs @@ -79,23 +79,26 @@ public override async Task GetVmNetworks(ManagedObjectReference mor foreach (ObjectContent obj in oc) { - // if (!obj.IsInPool(_client.pool)) - // continue; - string vmName = obj.GetProperty("name").ToString(); + VirtualMachineConfigInfo config = obj.GetProperty("config") as VirtualMachineConfigInfo; + foreach (VirtualEthernetCard card in config.hardware.device.OfType()) { if (card.backing is VirtualEthernetCardDistributedVirtualPortBackingInfo) { + var back = card.backing as VirtualEthernetCardDistributedVirtualPortBackingInfo; + result.Add(new VmNetwork { - NetworkMOR = ((VirtualEthernetCardDistributedVirtualPortBackingInfo)card.backing).port.portgroupKey, + NetworkMOR = $"DistributedVirtualPortgroup|{back.port.portgroupKey}", VmName = vmName }); } } + } + return result.ToArray(); } @@ -160,12 +163,13 @@ public override void UpdateEthernetCardBacking(VirtualEthernetCard card, string if (card.backing is VirtualEthernetCardDistributedVirtualPortBackingInfo) { string netMorName = this.Resolve(portgroupName); + card.backing = new VirtualEthernetCardDistributedVirtualPortBackingInfo { port = new DistributedVirtualSwitchPortConnection { switchUuid = _client.DvsUuid, - portgroupKey = netMorName + portgroupKey = netMorName.AsReference().Value } }; } diff --git a/src/TopoMojo.vSphere/HostNetworkManager.cs b/src/TopoMojo.vSphere/HostNetworkManager.cs index 0f8f73c7..666bf1bf 100644 --- a/src/TopoMojo.vSphere/HostNetworkManager.cs +++ b/src/TopoMojo.vSphere/HostNetworkManager.cs @@ -43,7 +43,7 @@ public override async Task AddPortGroup(string sw, VmNet et return new PortGroupAllocation { Net = eth.Net, - Key = "HostPortGroup|"+eth.Net, + Key = eth.Net, VlanId = eth.Vlan, Switch = sw }; @@ -104,7 +104,7 @@ public override async Task LoadPortGroups() list.Add(new PortGroupAllocation { Net = pg.spec.name, - Key = "HostPortGroup|" + pg.spec.name, + Key = pg.spec.name, VlanId = pg.spec.vlanId, Switch = pg.spec.vswitchName }); @@ -114,7 +114,7 @@ public override async Task LoadPortGroups() public override async Task RemovePortgroup(string pgReference) { - await _client.vim.RemovePortGroupAsync(_client.net, pgReference.AsReference().Value); + await _client.vim.RemovePortGroupAsync(_client.net, pgReference); } public override async Task RemoveSwitch(string sw) @@ -129,7 +129,6 @@ public override void UpdateEthernetCardBacking(VirtualEthernetCard card, string if (card.backing is VirtualEthernetCardNetworkBackingInfo) ((VirtualEthernetCardNetworkBackingInfo)card.backing).deviceName = portgroupName; - card.connectable = new VirtualDeviceConnectInfo() { connected = true, diff --git a/src/TopoMojo.vSphere/HypervisorService.cs b/src/TopoMojo.vSphere/HypervisorService.cs index 3397a34a..bbc60c68 100644 --- a/src/TopoMojo.vSphere/HypervisorService.cs +++ b/src/TopoMojo.vSphere/HypervisorService.cs @@ -30,6 +30,7 @@ ILoggerFactory mill _vmCache = new ConcurrentDictionary(); _vlanman = new VlanManager(_options.Vlan); + NormalizeOptions(_options); } private readonly HypervisorServiceConfiguration _options; @@ -43,6 +44,7 @@ ILoggerFactory mill private ConcurrentDictionary _vmCache; public HypervisorServiceConfiguration Options { get {return _options;}} + public async Task ReloadHost(string hostname) { string host = "https://" + hostname + "/sdk"; @@ -92,9 +94,9 @@ public async Task Deploy(VmTemplate template) if (vm != null) return vm; - _logger.LogDebug("deploy: find host "); VimClient host = FindHostByAffinity(template.IsolationTag); _logger.LogDebug("deploy: host " + host.Name); + NormalizeTemplate(template, host.Options); _logger.LogDebug("deploy: normalized "+ template.Name); @@ -105,8 +107,11 @@ public async Task Deploy(VmTemplate template) throw new Exception("Template disks have not been prepared."); } - _logger.LogDebug("deploy: reserve vlans "); - _vlanman.ReserveVlans(template, host.Options.IsVCenter); + if (!host.Options.Uplink.StartsWith("nsx.")) + { + _logger.LogDebug("deploy: reserve vlans "); + _vlanman.ReserveVlans(template, host.Options.IsVCenter); + } _logger.LogDebug("deploy: " + template.Name + " " + host.Name); return await host.Deploy(template); @@ -127,7 +132,8 @@ public async Task Load(string id) Vm vm = _vmCache.Values.Where(o=>o.Id == id || o.Name == id).FirstOrDefault(); - CheckProgress(vm); + if (vm != null) + CheckProgress(vm); return vm; } @@ -464,6 +470,7 @@ private void NormalizeTemplate(VmTemplate template, HypervisorServiceConfigurati // need to have a backing file to add the cdrom device template.Iso = option.IsoStore + "null.iso"; } + var isopath = new DatastorePath(template.Iso); isopath.Merge(option.IsoStore); template.Iso = isopath.ToString(); @@ -643,6 +650,20 @@ protected class HostVmCount public string Name { get; set; } public int Count { get; set; } } + + private void NormalizeOptions(HypervisorServiceConfiguration options) + { + var regex = new Regex("(]|/)$"); + + if (!regex.IsMatch(options.VmStore)) + options.VmStore += "/"; + + if (!regex.IsMatch(options.DiskStore)) + options.DiskStore += "/"; + + if (!regex.IsMatch(options.IsoStore)) + options.IsoStore += "/"; + } } } diff --git a/src/TopoMojo.vSphere/MockHypervisorService.cs b/src/TopoMojo.vSphere/MockHypervisorService.cs index f1245f54..39366840 100644 --- a/src/TopoMojo.vSphere/MockHypervisorService.cs +++ b/src/TopoMojo.vSphere/MockHypervisorService.cs @@ -27,6 +27,8 @@ ILoggerFactory mill _vms = new Dictionary(); _tasks = new Dictionary(); _rand = new Random(); + + NormalizeOptions(_optPod); } private readonly HypervisorServiceConfiguration _optPod; @@ -295,8 +297,9 @@ public async Task VerifyDisks(VmTemplate template) VmDisk disk = template.Disks.FirstOrDefault(); if (disk != null) { - if (disk.Path.Contains("blank-")) - return 100; + + if (disk.Path.Contains("blank-")) + return 100; MockDisk mock = _disks.FirstOrDefault(o=>o.Path == disk.Path); if (mock == null) @@ -361,6 +364,51 @@ public string Version } private void NormalizeTemplate(VmTemplate template, HypervisorServiceConfiguration option) + { + if (!template.Iso.HasValue()) + { + // need to have a backing file to add the cdrom device + template.Iso = option.IsoStore + "null.iso"; + } + + var isopath = new DatastorePath(template.Iso); + isopath.Merge(option.IsoStore); + template.Iso = isopath.ToString(); + + foreach (VmDisk disk in template.Disks) + { + if (!disk.Path.StartsWith(option.DiskStore) + ) { + DatastorePath dspath = new DatastorePath(disk.Path); + dspath.Merge(option.DiskStore); + disk.Path = dspath.ToString(); + } + if (disk.Source.HasValue() && !disk.Source.StartsWith(option.DiskStore) + ) { + DatastorePath dspath = new DatastorePath(disk.Source); + dspath.Merge(option.DiskStore); + disk.Source = dspath.ToString(); + } + } + + if (template.IsolationTag.HasValue()) + { + string tag = "#" + template.IsolationTag; + Regex rgx = new Regex("#.*"); + if (!template.Name.EndsWith(template.IsolationTag)) + template.Name = rgx.Replace(template.Name, "") + tag; + foreach (VmNet eth in template.Eth) + { + // //don't add tag if referencing a global vlan + // if (!_vlanman.Contains(eth.Net)) + // { + // eth.Net = rgx.Replace(eth.Net, "") + tag; + // } + } + } + } + + private void NormalizeTemplateOld(VmTemplate template, HypervisorServiceConfiguration option) { if (template.Iso.HasValue() && !template.Iso.StartsWith(option.IsoStore)) { @@ -466,6 +514,20 @@ public Task StopAll(string target) { throw new NotImplementedException(); } + + private void NormalizeOptions(HypervisorServiceConfiguration options) + { + var regex = new Regex("(]|/)$"); + + if (!regex.IsMatch(options.VmStore)) + options.VmStore += "/"; + + if (!regex.IsMatch(options.DiskStore)) + options.DiskStore += "/"; + + if (!regex.IsMatch(options.IsoStore)) + options.IsoStore += "/"; + } } public class MockDisk diff --git a/src/TopoMojo.vSphere/NetworkManager.cs b/src/TopoMojo.vSphere/NetworkManager.cs index 6840d626..d0ffddd5 100644 --- a/src/TopoMojo.vSphere/NetworkManager.cs +++ b/src/TopoMojo.vSphere/NetworkManager.cs @@ -58,10 +58,14 @@ public async Task Initialize() //process vm counts var map = GetKeyMap(); + var vmnets = await GetVmNetworks(_client.pool); + foreach (var vmnet in vmnets) + { if (map.ContainsKey(vmnet.NetworkMOR)) map[vmnet.NetworkMOR].Counter += 1; + } //remove empties await Clean(); @@ -70,6 +74,7 @@ public async Task Initialize() public async Task Provision(VmTemplate template) { await Task.Delay(0); + lock (_pgAllocation) { string sw = _client.UplinkSwitch; @@ -88,16 +93,31 @@ public async Task Provision(VmTemplate template) if (!_pgAllocation.ContainsKey(eth.Net)) { var pg = AddPortGroup(sw, eth).Result; + pg.Counter = 1; + _pgAllocation.Add(pg.Net, pg); - _vlanManager.Activate(new Vlan[] { new Vlan { Id = pg.VlanId, Name = pg.Net, OnUplink = sw == _client.UplinkSwitch }}); + + if (pg.VlanId > 0) + { + _vlanManager.Activate(new Vlan[] { + new Vlan { + Id = pg.VlanId, + Name = pg.Net, + OnUplink = sw == _client.UplinkSwitch + } + }); + } + if (_swAllocation.ContainsKey(sw)) _swAllocation[sw] += 1; + } else { _pgAllocation[eth.Net].Counter += 1; } - eth.Net = _pgAllocation[eth.Net].Key.AsReference().Value; + + eth.Key = _pgAllocation[eth.Net].Key; } } } @@ -105,10 +125,13 @@ public async Task Provision(VmTemplate template) public async Task Unprovision(ManagedObjectReference vmMOR) { await Task.Delay(0); + lock(_pgAllocation) { var map = GetKeyMap(); + var vmnets = GetVmNetworks(vmMOR).Result; + foreach (var vmnet in vmnets) if (map.ContainsKey(vmnet.NetworkMOR)) map[vmnet.NetworkMOR].Counter -= 1; @@ -139,8 +162,11 @@ public async Task Clean(string tag, bool all) ) { RemovePortgroup(pg.Key).Wait(); + _pgAllocation.Remove(pg.Net); + _vlanManager.Deactivate(pg.Net); + if (_swAllocation.ContainsKey(pg.Switch)) _swAllocation[pg.Switch] -= 1; } @@ -151,6 +177,7 @@ public async Task Clean(string tag, bool all) if (_swAllocation[sw] < 1 && sw.Contains("#")) { RemoveSwitch(sw).Wait(); + _swAllocation.Remove(sw); } } @@ -163,9 +190,8 @@ private Dictionary GetKeyMap() var map = new Dictionary(); foreach (var pga in _pgAllocation.Values) { - string key = pga.Key.AsReference().Value; - if (!map.ContainsKey(key)) - map.Add(key, pga); + if (!map.ContainsKey(pga.Key)) + map.Add(pga.Key, pga); } return map; } @@ -204,7 +230,7 @@ protected async Task WaitForVimTask(ManagedObjectReference task) public string Resolve(string net) { - return _pgAllocation[net]?.Key.AsReference().Value ?? "notfound"; + return _pgAllocation[net]?.Key ?? "notfound"; } public abstract void UpdateEthernetCardBacking(VirtualEthernetCard card, string portgroupName); diff --git a/src/TopoMojo.vSphere/NsxNetworkManager.cs b/src/TopoMojo.vSphere/NsxNetworkManager.cs new file mode 100644 index 00000000..232cfeb2 --- /dev/null +++ b/src/TopoMojo.vSphere/NsxNetworkManager.cs @@ -0,0 +1,249 @@ +// Copyright 2020 Carnegie Mellon University. All Rights Reserved. +// Released under a 3 Clause BSD-style license. See LICENSE.md in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using NetVimClient; +using TopoMojo.Models; +using TopoMojo.vSphere.Helpers; + +namespace TopoMojo.vSphere +{ + public class NsxNetworkManager : NetworkManager + { + public NsxNetworkManager( + VimReferences settings, + ConcurrentDictionary vmCache, + VlanManager vlanManager, + SddcConfiguration sddcConfig + ) : base(settings, vmCache, vlanManager) + { + _config = sddcConfig; + } + + private readonly SddcConfiguration _config; + private HttpClient _sddc; + private DateTime authExpiration = DateTime.MinValue; + private string authToken = ""; + private string _apiUrl = ""; + private string _apiSegments = "policy/api/v1/infra/tier-1s/cgw/segments"; + + private async Task InitClient() + { + if (DateTime.UtcNow.CompareTo(authExpiration) < 0) + return; + + _sddc = new HttpClient(); + + var content = new FormUrlEncodedContent( + new KeyValuePair[] { + new KeyValuePair( + "refresh_token", + _config.ApiKey + ) + } + ); + + var response = await _sddc.PostAsync( + _config.AuthUrl, + content + ); + + if (!response.IsSuccessStatusCode) + throw new InvalidOperationException("SDDC login failed."); + + string data = await response.Content.ReadAsStringAsync(); + var auth = JsonSerializer.Deserialize(data); + + authExpiration = DateTime.UtcNow.AddSeconds(auth.expires_in); + + _sddc.DefaultRequestHeaders.Add("csp-auth-token", auth.access_token); + + string meta = await _sddc.GetStringAsync(_config.Url); + + var sddc = JsonSerializer.Deserialize(meta); + + _apiUrl = sddc.resource_config.nsx_api_public_endpoint_url; + + } + + public override async Task AddPortGroup(string sw, VmNet eth) + { + await InitClient(); + + string url = $"{_apiUrl}/{_apiSegments}/{eth.Net.Replace("#","%23")}"; + + var response = await _sddc.PutAsync( + url, + new StringContent( + "{\"advanced_config\": { \"connectivity\": \"OFF\" } }", + Encoding.UTF8, + "application/json" + ) + ); + + int count = 0; + PortGroupAllocation pga = null; + + while (pga == null && count < 10) + { + // slight delay + await Task.Delay(1500); + + count += 1; + + pga = (await LoadPortGroups()) + .FirstOrDefault(p => p.Net == eth.Net); + + } + + if (pga == null) + throw new Exception($"Failed to create net {eth.Net}"); + + return pga; + + } + + public override Task AddSwitch(string sw) + { + return Task.FromResult(0); + } + + public override async Task GetVmNetworks(ManagedObjectReference mor) + { + var result = new List(); + RetrievePropertiesResponse response = await _client.vim.RetrievePropertiesAsync( + _client.props, + FilterFactory.VmFilter(mor, "name config")); + ObjectContent[] oc = response.returnval; + + foreach (ObjectContent obj in oc) + { + string vmName = obj.GetProperty("name").ToString(); + VirtualMachineConfigInfo config = obj.GetProperty("config") as VirtualMachineConfigInfo; + foreach (VirtualEthernetCard card in config.hardware.device.OfType()) + { + if (card.backing is VirtualEthernetCardOpaqueNetworkBackingInfo) + { + var back = card.backing as VirtualEthernetCardOpaqueNetworkBackingInfo; + + result.Add(new VmNetwork + { + NetworkMOR = $"{back.opaqueNetworkType}#{back.opaqueNetworkId}", + VmName = vmName + }); + } + } + } + return result.ToArray(); + } + + public override async Task LoadPortGroups() + { + var list = new List(); + + RetrievePropertiesResponse response = await _client.vim.RetrievePropertiesAsync( + _client.props, + FilterFactory.OpaqueNetworkFilter(_client.cluster)); + + ObjectContent[] clunkyTree = response.returnval; + foreach (var dvpg in clunkyTree.FindType("OpaqueNetwork")) + { + var config = (OpaqueNetworkSummary)dvpg.GetProperty("summary"); + + if (Regex.Match(config.name, _client.ExcludeNetworkMask).Success) + continue; + + list.Add( + new PortGroupAllocation + { + Net = config.name, + Key = $"{config.opaqueNetworkType}#{config.opaqueNetworkId}", + Switch = config.opaqueNetworkType + } + ); + } + + return list.ToArray(); + } + + public override async Task RemovePortgroup(string pgReference) + { + try + { + var pga = _pgAllocation.Values.FirstOrDefault(v => v.Key == pgReference); + + if (pga == null || !pga.Net.Contains("#")) + return; + + await InitClient(); + + // slight delay + await Task.Delay(1500); + + var response = await _sddc.DeleteAsync( + $"{_apiUrl}/{_apiSegments}/{pga.Net.Replace("#","%23")}" + ); + + if (!response.IsSuccessStatusCode) + { + Console.WriteLine("error removing net"); + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + } + + public override Task RemoveSwitch(string sw) + { + return Task.FromResult(0); + } + + public override void UpdateEthernetCardBacking(VirtualEthernetCard card, string portgroupName) + { + if (card != null) + { + if (card.backing is VirtualEthernetCardOpaqueNetworkBackingInfo) + { + string netMorName = this.Resolve(portgroupName); + card.backing = new VirtualEthernetCardOpaqueNetworkBackingInfo + { + opaqueNetworkId = netMorName.Tag(), + opaqueNetworkType = netMorName.Untagged() + }; + } + + card.connectable = new VirtualDeviceConnectInfo() + { + connected = true, + startConnected = true, + }; + } + } + + internal class AuthResponse + { + public string access_token { get; set; } + public int expires_in { get; set; } + } + + internal class SddcResponse + { + public SddcResourceConfig resource_config { get; set; } + } + + internal class SddcResourceConfig + { + public string nsx_api_public_endpoint_url { get; set; } + } + } +} diff --git a/src/TopoMojo.vSphere/TopoMojo.vSphere.csproj b/src/TopoMojo.vSphere/TopoMojo.vSphere.csproj index f5b03261..fdca319e 100755 --- a/src/TopoMojo.vSphere/TopoMojo.vSphere.csproj +++ b/src/TopoMojo.vSphere/TopoMojo.vSphere.csproj @@ -16,6 +16,7 @@ + diff --git a/src/TopoMojo.vSphere/VimClient.cs b/src/TopoMojo.vSphere/VimClient.cs index 1bf0b062..94f8ae2b 100644 --- a/src/TopoMojo.vSphere/VimClient.cs +++ b/src/TopoMojo.vSphere/VimClient.cs @@ -12,6 +12,7 @@ using NetVimClient; using TopoMojo.Models; using TopoMojo.vSphere.Helpers; +using System.IO; namespace TopoMojo.vSphere { @@ -42,14 +43,14 @@ ILogger logger private ConcurrentDictionary _vmCache; private Dictionary _pgAllocation; Dictionary _taskMap = new Dictionary(); - + Dictionary _dsnsMap = new Dictionary(); private INetworkManager _netman; HypervisorServiceConfiguration _config = null; VimPortTypeClient _vim = null; ServiceContent _sic = null; UserSession _session = null; ManagedObjectReference _props, _vdm, _file; - ManagedObjectReference _datacenter, _vms, _res, _pool, _dvs; + ManagedObjectReference _datacenter, _dsns, _vms, _res, _pool, _dvs; string _dvsuuid = ""; int _pollInterval = 1000; int _syncInterval = 30000; @@ -452,7 +453,7 @@ public async Task FolderExists(string path) public async Task FileExists(string path) { string[] list = await GetFiles(path, false); - return list.Length > 0; + return list.Any(x => x == path); } public async Task GetFiles(string path, bool recursive) @@ -460,39 +461,98 @@ public async Task GetFiles(string path, bool recursive) await Connect(); List list = new List(); DatastorePath dsPath = new DatastorePath(path); + string oldRoot = ""; + string pattern = dsPath.File ?? "*"; RetrievePropertiesResponse response = await _vim.RetrievePropertiesAsync( - _props, FilterFactory.DatastoreFilter(_res)); + _props, + FilterFactory.DatastoreFilter(_res) + ); + ObjectContent[] oc = response.returnval; foreach (ObjectContent obj in oc) { ManagedObjectReference dsBrowser = (ManagedObjectReference)obj.propSet[0].val; - string dsName = ((DatastoreSummary)obj.propSet[1].val).name; - if (dsName == dsPath.Datastore) + + var capability = obj.propSet[1].val as DatastoreCapability; + + var summary = obj.propSet[2].val as DatastoreSummary; + + // if topLevelDirectory not supported (vsan), map from directory name to guid) + if ( + capability.topLevelDirectoryCreateSupportedSpecified + && !capability.topLevelDirectoryCreateSupported + && dsPath.TopLevelFolder.HasValue() + ) + { + oldRoot = dsPath.TopLevelFolder; + string target = summary.url + oldRoot; + + if (!_dsnsMap.ContainsKey(target)) + { + var result = await _vim.ConvertNamespacePathToUuidPathAsync( + _dsns, + _datacenter, + target + ); + + _dsnsMap.Add(target, result.Replace(summary.url, "")); + } + + dsPath.TopLevelFolder = _dsnsMap[target]; + + // vmcloud sddc errors on Search_Datastore() + // so force SearchDatastoreSubFolders() + recursive = true; + pattern = "*" + Path.GetExtension(dsPath.File); + + _logger.LogDebug("mapped datastore namespace: " + dsPath.ToString()); + + } + + if (summary.name == dsPath.Datastore) { ManagedObjectReference task = null; TaskInfo info = null; - HostDatastoreBrowserSearchSpec spec = new HostDatastoreBrowserSearchSpec + + var spec = new HostDatastoreBrowserSearchSpec { - matchPattern = new string[] { dsPath.File } + matchPattern = new string[] { pattern }, }; - List results = new List(); + + var results = new List(); + if (recursive) { - task = await _vim.SearchDatastoreSubFolders_TaskAsync( - dsBrowser, dsPath.FolderPath, spec); - info = await WaitForVimTask(task); - if (info.result != null) - results.AddRange((HostDatastoreBrowserSearchResults[])info.result); + try { + + task = await _vim.SearchDatastoreSubFolders_TaskAsync( + dsBrowser, dsPath.FolderPath, spec + ); + + info = await WaitForVimTask(task); + + if (info.result != null) + results.AddRange((HostDatastoreBrowserSearchResults[])info.result); + + } + catch (Exception ex) + { + _logger.LogError(ex, "error searching datastore."); + } } else { task = await _vim.SearchDatastore_TaskAsync( - dsBrowser, dsPath.FolderPath, spec); + dsBrowser, dsPath.FolderPath, spec + ); + info = await WaitForVimTask(task); + if (info.result != null) results.Add((HostDatastoreBrowserSearchResults)info.result); + } foreach (HostDatastoreBrowserSearchResults result in results) @@ -500,6 +560,9 @@ public async Task GetFiles(string path, bool recursive) if (result != null && result.file != null && result.file.Length > 0) { string fp = result.folderPath; + if (oldRoot.HasValue()) + fp = fp.Replace(dsPath.TopLevelFolder, oldRoot); + if (!fp.EndsWith("/")) fp += "/"; @@ -668,19 +731,21 @@ private async Task GetVimTaskInfo(ManagedObjectReference task) TaskInfo info = new TaskInfo(); - RetrievePropertiesResponse response = await _vim.RetrievePropertiesAsync( - _props, - FilterFactory.TaskFilter(task)); - - ObjectContent[] oc = response.returnval; - try { + RetrievePropertiesResponse response = await _vim.RetrievePropertiesAsync( + _props, + FilterFactory.TaskFilter(task) + ); + + ObjectContent[] oc = response.returnval; + info = (TaskInfo)oc[0]?.propSet[0]?.val; } - catch + catch (Exception ex) { - _logger.LogDebug($"Failed to get TaskInfo {task.Value}"); + _logger.LogError(ex, "Failed to get TaskInfo for {0}", task.Value); + info = new TaskInfo { task = task, state = TaskInfoState.error }; } @@ -721,6 +786,8 @@ private async Task Connect() _props = _sic.propertyCollector; _vdm = _sic.virtualDiskManager; _file = _sic.fileManager; + _dsns = _sic.datastoreNamespaceManager; + _logger.LogDebug($"Connected {_config.Host} in {DateTime.Now.Subtract(sp).TotalSeconds} seconds"); sp = DateTime.Now; @@ -907,18 +974,31 @@ private async Task InitReferences(VimPortTypeClient client) if (subpools != null && subpools.Length > 0) _pool = subpools.First(); - var dvs = clunkyTree.FindTypeByName("DistributedVirtualSwitch", _config.Uplink.ToLower()) ?? clunkyTree.First("DistributedVirtualSwitch"); - _dvs = dvs?.obj; - _dvsuuid = dvs?.GetProperty("uuid").ToString(); + if (_config.Uplink.StartsWith("nsx.")) + { + _netman = new NsxNetworkManager( + netSettings, + _vmCache, + _vlanman, + _config.Sddc + ); + } + else + { - netSettings.dvs = dvs?.obj; - netSettings.DvsUuid = _dvsuuid; + var dvs = clunkyTree.FindTypeByName("DistributedVirtualSwitch", _config.Uplink.ToLower()) ?? clunkyTree.First("DistributedVirtualSwitch"); + _dvs = dvs?.obj; + _dvsuuid = dvs?.GetProperty("uuid").ToString(); - _netman = new DistributedNetworkManager( - netSettings, - _vmCache, - _vlanman - ); + netSettings.dvs = dvs?.obj; + netSettings.DvsUuid = _dvsuuid; + + _netman = new DistributedNetworkManager( + netSettings, + _vmCache, + _vlanman + ); + } } else { diff --git a/src/TopoMojo.vSphere/VimFilters.cs b/src/TopoMojo.vSphere/VimFilters.cs index 8380ee23..2a00349c 100755 --- a/src/TopoMojo.vSphere/VimFilters.cs +++ b/src/TopoMojo.vSphere/VimFilters.cs @@ -86,7 +86,7 @@ public static PropertyFilterSpec[] DatastoreFilter(ManagedObjectReference mor) { PropertySpec prop = new PropertySpec { type = "Datastore", - pathSet = new string[] { "browser", "summary" } + pathSet = new string[] { "browser", "capability", "summary" } }; ObjectSpec objectspec = new ObjectSpec { @@ -152,6 +152,32 @@ public static PropertyFilterSpec[] DistributedPortgroupFilter(ManagedObjectRefer }; } + public static PropertyFilterSpec[] OpaqueNetworkFilter(ManagedObjectReference mor) + { + PropertySpec prop = new PropertySpec { + type = "OpaqueNetwork", + pathSet = new string[] { "summary" } + }; + + ObjectSpec objectspec = new ObjectSpec { + obj = mor, //_pool + selectSet = new SelectionSpec[] + { + new TraversalSpec { + type = "ComputeResource", + path = "network", + } + } + }; + + return new PropertyFilterSpec[] { + new PropertyFilterSpec { + propSet = new PropertySpec[] { prop }, + objectSet = new ObjectSpec[] { objectspec } + } + }; + } + public static PropertyFilterSpec[] InitFilter(ManagedObjectReference rootMOR) { var plan = new TraversalSpec diff --git a/src/TopoMojo.vSphere/VimTransform.cs b/src/TopoMojo.vSphere/VimTransform.cs index b73aa939..df94404d 100755 --- a/src/TopoMojo.vSphere/VimTransform.cs +++ b/src/TopoMojo.vSphere/VimTransform.cs @@ -124,20 +124,29 @@ private static VirtualDeviceConfigSpec GetEthernetAdapter(ref int key, VmNet nic if (nic.Type == "e1000e") eth = new VirtualE1000e(); - // VirtualEthernetCardNetworkBackingInfo ethbacking = new VirtualEthernetCardNetworkBackingInfo(); - // ethbacking.deviceName = nic.Net; - eth.key = key--; - if (dvsuuid.HasValue()) + + if (nic.Net.StartsWith("nsx.")) + { + eth.backing = new VirtualEthernetCardOpaqueNetworkBackingInfo { + opaqueNetworkId = nic.Key.Tag(), + opaqueNetworkType = nic.Key.Untagged() + }; + } + else if (dvsuuid.HasValue()) + { eth.backing = new VirtualEthernetCardDistributedVirtualPortBackingInfo { port = new DistributedVirtualSwitchPortConnection { switchUuid = dvsuuid, - portgroupKey = nic.Net + portgroupKey = nic.Key.AsReference().Value } }; + } else - eth.backing = new VirtualEthernetCardNetworkBackingInfo { deviceName = nic.Net }; + { + eth.backing = new VirtualEthernetCardNetworkBackingInfo { deviceName = nic.Key }; + } devicespec = new VirtualDeviceConfigSpec(); devicespec.device = eth; diff --git a/src/TopoMojo.vSphere/VlanManager.cs b/src/TopoMojo.vSphere/VlanManager.cs index 7b09f26d..d56ba85e 100644 --- a/src/TopoMojo.vSphere/VlanManager.cs +++ b/src/TopoMojo.vSphere/VlanManager.cs @@ -52,8 +52,12 @@ public void Activate(Vlan[] vlans) { foreach (Vlan vlan in vlans) { + if (vlan.Id == 0) + continue; + if (vlan.OnUplink) _vlanMap[vlan.Id] = true; + if (!_vlans.ContainsKey(vlan.Name)) _vlans.Add(vlan.Name, vlan); }