Add deployment status feature and cmdlet for deploying Yams applications

Etg.Yams.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 14
-VisualStudioVersion = 14.0.25420.1
+# Visual Studio 15
+VisualStudioVersion = 15.0.26730.12
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Etg.Yams", "src\Etg.Yams\Etg.Yams.csproj", "{E1A6854F-6C0D-4F93-9B8D-D6981727D15B}"
 EndProject
@@ -47,6 +47,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeartBeatProcess", "test\He
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FullIpcProcess", "test\FullIpcProcess\FullIpcProcess.csproj", "{99F3A36C-7930-4670-A8B3-7137D891671D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Etg.Yams.Powershell", "src\Etg.Yams.Powershell\Etg.Yams.Powershell.csproj", "{CDFBE3EE-AFD7-467F-BE9C-F9EAFC7A05CE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureTableDeploymentStatusManager", "AzureTableDeploymentStatusManager\AzureTableDeploymentStatusManager.csproj", "{8ACD9ED9-8C96-4AFD-B790-F5E836720539}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -133,6 +137,14 @@ Global
 		{99F3A36C-7930-4670-A8B3-7137D891671D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{99F3A36C-7930-4670-A8B3-7137D891671D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{99F3A36C-7930-4670-A8B3-7137D891671D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CDFBE3EE-AFD7-467F-BE9C-F9EAFC7A05CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CDFBE3EE-AFD7-467F-BE9C-F9EAFC7A05CE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CDFBE3EE-AFD7-467F-BE9C-F9EAFC7A05CE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CDFBE3EE-AFD7-467F-BE9C-F9EAFC7A05CE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8ACD9ED9-8C96-4AFD-B790-F5E836720539}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8ACD9ED9-8C96-4AFD-B790-F5E836720539}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8ACD9ED9-8C96-4AFD-B790-F5E836720539}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8ACD9ED9-8C96-4AFD-B790-F5E836720539}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -158,8 +170,11 @@ Global
 		{706E37D7-B148-4734-9695-0D1AE6D4B3D5} = {2A52BDC8-4B16-43FE-9E9D-A7D0A72C17C8}
 		{257044FB-F7A8-499D-8EF2-B5CAD76D617A} = {2A52BDC8-4B16-43FE-9E9D-A7D0A72C17C8}
 		{99F3A36C-7930-4670-A8B3-7137D891671D} = {2A52BDC8-4B16-43FE-9E9D-A7D0A72C17C8}
+		{CDFBE3EE-AFD7-467F-BE9C-F9EAFC7A05CE} = {5925E681-1BEA-456D-B9E0-CA175ABBFA9D}
+		{8ACD9ED9-8C96-4AFD-B790-F5E836720539} = {5925E681-1BEA-456D-B9E0-CA175ABBFA9D}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {7A9645D4-78F9-451C-B788-6EC41CA3BF59}
 		EnterpriseLibraryConfigurationToolBinariesPathV6 = packages\EnterpriseLibrary.TransientFaultHandling.6.0.1304.0\lib\portable-net45+win+wp8
 	EndGlobalSection
 EndGlobal

Etg.Yams.sln.DotSettings

@@ -0,0 +1,2 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateInstanceFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="_" Suffix="" Style="aaBb" /&gt;</s:String></wpf:ResourceDictionary>

src/AzureBlobStorageDeploymentRepository/BlobStorageDeploymentRepository.cs

@@ -8,30 +8,40 @@
 using Microsoft.WindowsAzure.Storage.Blob;
 using Etg.Yams.Json;
 using Newtonsoft.Json.Serialization;
+using Etg.Yams.Storage.Status;
+using System;
+using System.Collections.Generic;
+using Etg.Yams.Azure.Lease;
 
 namespace Etg.Yams.Azure.Storage
 {
-    public class BlobStorageDeploymentRepository : IDeploymentRepository
+    public class BlobStorageDeploymentRepository : IDeploymentRepository, IDeploymentStatusReader, IDeploymentStatusWriter
     {
         public const string ApplicationsRootFolderName = "applications";
         private readonly CloudBlobContainer _blobContainer;
-        private readonly IDeploymentConfigSerializer _serializer;
+        private readonly IDeploymentConfigSerializer _deploymentConfigSerializer;
+        private readonly IDeploymentStatusSerializer _deploymentStatusSerializer;
 
-        public BlobStorageDeploymentRepository(CloudBlobContainer blobContainer, IDeploymentConfigSerializer serializer)
+        public BlobStorageDeploymentRepository(CloudBlobContainer blobContainer, IDeploymentConfigSerializer serializer,
+            IDeploymentStatusSerializer deploymentStatusSerializer)
         {
             _blobContainer = blobContainer;
-            _serializer = serializer;
+            _deploymentConfigSerializer = serializer;
+            _deploymentStatusSerializer = deploymentStatusSerializer;
         }
 
-        public BlobStorageDeploymentRepository(string connectionString, IDeploymentConfigSerializer serializer) 
-            : this(GetApplicationsContainerReference(connectionString), serializer)
+        public BlobStorageDeploymentRepository(string connectionString, IDeploymentConfigSerializer deploymentConfigSerializer,
+            IDeploymentStatusSerializer deploymentStatusSerializer) 
+            : this(GetApplicationsContainerReference(connectionString), deploymentConfigSerializer, deploymentStatusSerializer)
         {
         }
 
         public static BlobStorageDeploymentRepository Create(string connectionString)
         {
-            IDeploymentConfigSerializer serializer = new JsonDeploymentConfigSerializer(new JsonSerializer(new DiagnosticsTraceWriter()));
-            return new BlobStorageDeploymentRepository(connectionString, serializer);
+            var jsonSerializer = new JsonSerializer(new DiagnosticsTraceWriter());
+            IDeploymentConfigSerializer deploymentConfigSerializer = new JsonDeploymentConfigSerializer(jsonSerializer);
+            IDeploymentStatusSerializer deploymentStatusSerializer = new JsonDeploymentStatusSerializer(jsonSerializer);
+            return new BlobStorageDeploymentRepository(connectionString, deploymentConfigSerializer, deploymentStatusSerializer);
         }
 
         private static CloudBlobContainer GetApplicationsContainerReference(string connectionString)
@@ -62,7 +72,7 @@ public async Task<DeploymentConfig> FetchDeploymentConfig()
             }
 
             string data = await blob.DownloadTextAsync();
-            return _serializer.Deserialize(data);
+            return _deploymentConfigSerializer.Deserialize(data);
         }
 
         public Task<bool> HasApplicationBinaries(AppIdentity appIdentity)
@@ -97,7 +107,7 @@ public async Task DownloadApplicationBinaries(AppIdentity appIdentity, string lo
         public Task PublishDeploymentConfig(DeploymentConfig deploymentConfig)
         {
             CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(Constants.DeploymentConfigFileName);
-            return blob.UploadTextAsync(_serializer.Serialize(deploymentConfig));
+            return blob.UploadTextAsync(_deploymentConfigSerializer.Serialize(deploymentConfig));
         }
 
         public async Task UploadApplicationBinaries(AppIdentity appIdentity, string localPath,
@@ -146,5 +156,78 @@ private string GetBlobDirectoryRelPath(AppIdentity appIdentity)
         {
             return appIdentity.Id + "/" + appIdentity.Version;
         }
+
+        public async Task<ClusterDeploymentStatus> FetchClusterDeploymentStatus(string clusterId, int ttlSeconds)
+        {
+            ClusterDeploymentStatus clusterDeploymentStatus = new ClusterDeploymentStatus();;
+            CloudBlobDirectory instancesBlobDir = _blobContainer.GetDirectoryReference(GetClusterStatusRelativePath(clusterId));
+            if (!await instancesBlobDir.ExistsAsync())
+            {
+                return clusterDeploymentStatus;
+            }
+            IEnumerable<IListBlobItem> instancesBlobList = await instancesBlobDir.ListBlobsAsync();
+            foreach (var blob in instancesBlobList)
+            {
+                CloudBlockBlob instanceBlob = blob as CloudBlockBlob;
+                if (instanceBlob == null)
+                {
+                    Trace.TraceWarning($"Unexpected blob {blob.Uri} in cluster status directory");
+                    continue;
+                }
+                double secondsSinceModified = DateTimeOffset.UtcNow.Subtract(instanceBlob.Properties.LastModified.Value).TotalSeconds;
+                if (secondsSinceModified > ttlSeconds)
+                {
+                    continue;
+                }
+
+                try
+                {
+                    InstanceDeploymentStatus instanceDeploymentStatus =
+                        await FetchInstanceDeploymentStatus(instanceBlob);
+                    string instanceId = instanceBlob.Name;
+                    clusterDeploymentStatus.SetInstanceDeploymentStatus(instanceId, instanceDeploymentStatus);
+                }
+                catch (Exception e)
+                {
+                    Trace.TraceError($"Could not read instance deployment status {instanceBlob.Uri}, Exception: {e}");
+                }
+            }
+            return clusterDeploymentStatus;
+        }
+
+        public async Task<InstanceDeploymentStatus> FetchInstanceDeploymentStatus(string clusterId, string instanceId)
+        {
+            CloudBlockBlob instanceBlob =
+                _blobContainer.GetBlockBlobReference(GetInstanceStatusRelativePath(clusterId, instanceId));
+            if (!await instanceBlob.ExistsAsync())
+            {
+                return new InstanceDeploymentStatus();
+            }
+            return await FetchInstanceDeploymentStatus(instanceBlob);
+        }
+
+        private static string GetClusterStatusRelativePath(string clusterId)
+        {
+            return $"status/clusters/{clusterId}/instances";
+        }
+
+        private static string GetInstanceStatusRelativePath(string clusterId, string instanceId)
+        {
+            return $"{GetClusterStatusRelativePath(clusterId)}/{instanceId}";
+        }
+
+        private async Task<InstanceDeploymentStatus> FetchInstanceDeploymentStatus(CloudBlockBlob instanceBlob)
+        {
+            string data = await instanceBlob.DownloadTextAsync();
+            return _deploymentStatusSerializer.Deserialize(data);
+        }
+
+        public Task PublishInstanceDeploymentStatus(string clusterId, string instanceId, 
+            InstanceDeploymentStatus instanceDeploymentStatus)
+        {
+            CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(GetInstanceStatusRelativePath(clusterId, instanceId));
+            string data = _deploymentStatusSerializer.Serialize(instanceDeploymentStatus);
+            return blob.UploadTextAsync(data);
+        }
     }
 }

src/AzureBlobStorageUpdateSession/AzureBlobStorageUpdateSession.csproj

@@ -61,22 +61,6 @@
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
-    <Reference Include="System.Reactive.Core, Version=2.2.5.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\Rx-Core.2.2.5\lib\net45\System.Reactive.Core.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="System.Reactive.Interfaces, Version=2.2.5.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\Rx-Interfaces.2.2.5\lib\net45\System.Reactive.Interfaces.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="System.Reactive.Linq, Version=2.2.5.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\Rx-Linq.2.2.5\lib\net45\System.Reactive.Linq.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="System.Reactive.PlatformServices, Version=2.2.5.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\Rx-PlatformServices.2.2.5\lib\net45\System.Reactive.PlatformServices.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
     <Reference Include="System.Spatial, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll</HintPath>
     </Reference>
@@ -88,10 +72,6 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="Lease\BlobLeaseFactory.cs" />
-    <Compile Include="Lease\IBlobLease.cs" />
-    <Compile Include="Lease\IBlobLeaseFactory.cs" />
-    <Compile Include="Lease\SelfRenewableBlobLease.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="UpdateSession\BlobBasedUpdateSessionManager.cs" />
     <Compile Include="UpdateSession\AzureBlobStorageUpdateSessionDiModule.cs" />

src/AzureBlobStorageUpdateSession/UpdateSession/UpdateBlobFactory.cs

@@ -22,7 +22,7 @@ public async Task<IUpdateBlob> TryLockUpdateBlob(string appId)
         {
             string updateBlobName = GetUpdateBlobName(appId);
             ICloudBlob blob = GetBlob(updateBlobName);
-            await CreateBlobIfNoneExists(blob);
+            await BlobUtils.CreateBlobIfNotExists(blob);
 
             UpdateBlob updateBlob = new UpdateBlob(blob, _blobLeaseFactory);
             bool locked = await updateBlob.TryLock();
@@ -42,13 +42,5 @@ private CloudBlockBlob GetBlob(string updateBlobName)
         {
             return _blobContainer.GetBlockBlobReference(updateBlobName);
         }
-
-        private static async Task CreateBlobIfNoneExists(ICloudBlob updateBlob)
-        {
-            if (!await updateBlob.ExistsAsync())
-            {
-                await BlobUtils.CreateEmptyBlob(updateBlob);
-            }
-        }
     }
 }

src/AzureBlobStorageUpdateSession/packages.config

@@ -7,11 +7,6 @@
   <package id="Microsoft.Data.OData" version="5.8.2" targetFramework="net451" />
   <package id="Microsoft.Data.Services.Client" version="5.8.2" targetFramework="net451" />
   <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net451" />
-  <package id="Rx-Core" version="2.2.5" targetFramework="net451" />
-  <package id="Rx-Interfaces" version="2.2.5" targetFramework="net451" />
-  <package id="Rx-Linq" version="2.2.5" targetFramework="net451" />
-  <package id="Rx-Main" version="2.2.5" targetFramework="net451" />
-  <package id="Rx-PlatformServices" version="2.2.5" targetFramework="net451" />
   <package id="System.ComponentModel.EventBasedAsync" version="4.0.11" targetFramework="net451" />
   <package id="System.Dynamic.Runtime" version="4.0.0" targetFramework="net451" />
   <package id="System.Linq.Queryable" version="4.0.0" targetFramework="net451" />

src/AzureUtils/AzureUtils.csproj

@@ -53,17 +53,38 @@
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Reactive.Core, Version=3.0.1000.0, Culture=neutral, PublicKeyToken=94bc3704cddfc263, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\System.Reactive.Core.3.1.1\lib\net45\System.Reactive.Core.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Reactive.Interfaces, Version=3.0.1000.0, Culture=neutral, PublicKeyToken=94bc3704cddfc263, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\System.Reactive.Interfaces.3.1.1\lib\net45\System.Reactive.Interfaces.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Reactive.Linq, Version=3.0.1000.0, Culture=neutral, PublicKeyToken=94bc3704cddfc263, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\System.Reactive.Linq.3.1.1\lib\net45\System.Reactive.Linq.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Reactive.PlatformServices, Version=3.0.1000.0, Culture=neutral, PublicKeyToken=94bc3704cddfc263, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\System.Reactive.PlatformServices.3.1.1\lib\net45\System.Reactive.PlatformServices.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Reactive.Windows.Threading, Version=3.0.1000.0, Culture=neutral, PublicKeyToken=94bc3704cddfc263, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\System.Reactive.Windows.Threading.3.1.1\lib\net45\System.Reactive.Windows.Threading.dll</HintPath>
+    </Reference>
     <Reference Include="System.Spatial, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll</HintPath>
     </Reference>
+    <Reference Include="System.Windows" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="System.Data" />
     <Reference Include="System.Net.Http" />
     <Reference Include="System.Xml" />
+    <Reference Include="WindowsBase" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Lease\BlobLeaseFactory.cs" />
+    <Compile Include="Lease\IBlobLease.cs" />
+    <Compile Include="Lease\IBlobLeaseFactory.cs" />
+    <Compile Include="Lease\SelfRenewableBlobLease.cs" />
     <Compile Include="Utils\BlobUtils.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>

src/AzureUtils/Utils/BlobUtils.cs

@@ -54,7 +54,6 @@ public static Task<IEnumerable<IListBlobItem>> ListBlobsAsync(this CloudBlobDire
             return ListBlobsFlat(blobDirectory, useFlatBlobListing);
         }
 
-
         private static string GetLocalRelativePath(ICloudBlob blob, dynamic blobDirectory)
         {
             return GetBlobRelativePath(blob, blobDirectory).Replace('/', '\\');
@@ -75,6 +74,22 @@ public static string GetBlobRelativePath(ICloudBlob blob, CloudBlobContainer con
             return GetBlobRelativePathInternal(blob, container);
         }
 
+        public static async Task CreateBlobIfNotExists(ICloudBlob blob)
+        {
+            var emptyByteArray = new byte[] { };
+            try
+            {
+                await blob.UploadFromByteArrayAsync(emptyByteArray, 0, emptyByteArray.Length,
+                    AccessCondition.GenerateIfNotExistsCondition(), new BlobRequestOptions(), new OperationContext());
+            } catch(StorageException e)
+            {
+                if(e.RequestInformation.HttpStatusCode != 409)
+                {
+                    throw;
+                }
+            }
+        }
+
         public static async Task CreateEmptyBlob(ICloudBlob blob)
         {
             var emptyByteArray = new byte[] {};
@@ -112,24 +127,21 @@ public static CloudBlobContainer GetBlobContainer(string connectionString, strin
             return blobContainer;
         }
 
-        public static Task<bool> ExistsAsync(this CloudBlobDirectory dir)
+        public static async Task<bool> ExistsAsync(this CloudBlobDirectory dir)
         {
-            return Task.Run(() => dir.ListBlobs().Any());
+            return (await dir.ListBlobsAsync()).Any();
         }
 
-        public static Task DeleteAsync(this CloudBlobDirectory dir)
+        public static async Task DeleteAsync(this CloudBlobDirectory dir)
         {
-            return Task.Run(() =>
+            IEnumerable<IListBlobItem> blobs = await dir.ListBlobsAsync(useFlatBlobListing:true);
+            var tasks = new List<Task>();
+            foreach (IListBlobItem blobItem in blobs)
             {
-                IEnumerable<IListBlobItem> blobs = dir.ListBlobs(true);
-                var tasks = new List<Task>();
-                foreach (IListBlobItem blobItem in blobs)
-                {
-                    CloudBlockBlob blob = (CloudBlockBlob) blobItem;
-                    tasks.Add(blob.DeleteAsync());
-                }
-                return Task.WhenAll(tasks);
-            });
+                CloudBlockBlob blob = (CloudBlockBlob) blobItem;
+                tasks.Add(blob.DeleteAsync());
+            }
+            await Task.WhenAll(tasks);
         }
     }
 }