Merge pull request #33 from LuccaSA/gcp.cloud.storage.support

Google cloud storage support

.github/workflows/dotnetcore.yml

@@ -50,6 +50,8 @@ jobs:
         AZ_SAS_TOKEN: ${{ secrets.AZ_SAS_TOKEN }}
         AZ_URL: ${{ secrets.AZ_URL }}
         AZ_CONTAINER: ${{ secrets.AZ_CONTAINER }}
+        GoogleBucket: ${{ secrets.GoogleBucket }}
+        GoogleCredential: ${{ secrets.GoogleCredential }}
 
     - name: Merge coverages
       run: reportgenerator -reports:./coverage/*/*.xml -targetdir:./sonarCoverage -reporttypes:SonarQube

TCC.Lib/Dependencies/ExternalDependencies.cs

@@ -22,7 +22,6 @@ public ExternalDependencies(ILogger<ExternalDependencies> logger)
         public string Brotli() => GetPathEscaped(_brotli);
         public string Zstd() => GetPathEscaped(_zStd);
         public string OpenSsl() => GetPathEscaped(_openSsl);
-        public string AzCopy() => GetPathEscaped(_azCopy);
 
         public async Task EnsureAllDependenciesPresent()
         {
@@ -31,7 +30,6 @@ public async Task EnsureAllDependenciesPresent()
             await EnsureDependency(_zStd);
             await EnsureDependency(_openSsl);
             await EnsureDependency(_tar);
-            await EnsureDependency(_azCopy);
         }
 
         private string Root => Path.GetDirectoryName(AppContext.BaseDirectory);
@@ -154,14 +152,5 @@ private async Task DownloadDependencyInternal(Dependency dependency, string path
             ExtractFolder = "tar_v130",
             ExeName = "tar.exe"
         };
-
-        internal static readonly Dependency _azCopy = new Dependency
-        {
-            Name = "azcopy",
-            Url = @"https://azcopyvnext.azureedge.net/release20210415/azcopy_windows_amd64_10.10.0.zip",
-            ZipFilename = "azcopy_windows_amd64_10.10.0.zip",
-            ExtractFolder = "azcopy_windows_amd64_10.10.0",
-            ExeName = "azcopy_windows_amd64_10.10.0\\azcopy.exe"
-        };
     }
 }

TCC.Lib/Dependencies/UploadHelper.cs

@@ -0,0 +1,18 @@
+using System;
+using System.IO;
+
+namespace TCC.Lib.Dependencies
+{
+    public static class UploadHelper
+    {
+        public static string GetRelativeTargetPathTo(this FileInfo file, DirectoryInfo root )
+        {
+            var targetPath = file.FullName.Replace(root.FullName, string.Empty, StringComparison.InvariantCultureIgnoreCase)
+                .Trim('\\').Trim('/');
+
+            targetPath = targetPath.Replace('\\', '/');
+
+            return targetPath;
+        }
+    }
+}

TCC.Lib/Helpers/TccRegisteringExtensions.cs

@@ -24,7 +24,6 @@ public static void AddTcc(this IServiceCollection services, string workingPath =
             services.AddScoped<TarCompressCrypt>();
             services.AddScoped<EncryptionCommands>();
             services.AddScoped<CompressionCommands>();
-            services.AddScoped<UploadCommands>();
             services.AddScoped<BenchmarkRunner>();
             services.AddScoped<BenchmarkOptionHelper>();
             services.AddScoped<BenchmarkIterationGenerator>();

TCC.Lib/ObjectStorage/IObjectRemoteServer.cs

@@ -0,0 +1,134 @@
+using Azure.Storage.Blobs;
+using Google.Apis.Auth.OAuth2;
+using Google.Cloud.Storage.V1;
+using System;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using TCC.Lib.Dependencies;
+using TCC.Lib.Options;
+using Object = Google.Apis.Storage.v1.Data.Object;
+
+namespace TCC.Lib.ObjectStorage
+{
+    public interface IObjectStorageRemoteServer
+    {
+        Task<UploadResponse> UploadAsync(string targetPath, Stream data, CancellationToken token);
+
+        public async Task<UploadResponse> UploadAsync(FileInfo file, DirectoryInfo rootFolder, CancellationToken token)
+        {
+            string targetPath = file.GetRelativeTargetPathTo(rootFolder);
+            await using FileStream uploadFileStream = File.OpenRead(file.FullName);
+            return await UploadAsync(targetPath, uploadFileStream, token);
+        }
+    }
+
+    public static class RemoteServerFactory
+    {
+        public static async Task<IObjectStorageRemoteServer> CreateRemoteServerAsync(this CompressOption option, CancellationToken token)
+        {
+            switch (option.UploadMode)
+            {
+                case UploadMode.AzureSdk:
+                    var client = new BlobServiceClient(new Uri(option.AzBlobUrl + "/" + option.AzBlobContainer + "?" + option.AzSaS));
+                    BlobContainerClient container = client.GetBlobContainerClient(option.AzBlobContainer);
+                    return new AzureRemoteServer(container);
+                case UploadMode.GoogleCloudStorage:
+                    StorageClient storage = await GetGoogleStorageClient(option, token);
+                    return new GoogleRemoteServer(storage, option.GoogleStorageBucketName);
+                case UploadMode.None:
+                case null:
+                    return new NoneRemoteServer();
+                default:
+                    throw new ArgumentOutOfRangeException();
+            }
+        }
+
+        private static async Task<StorageClient> GetGoogleStorageClient(CompressOption option, CancellationToken token)
+        {
+            GoogleCredential credential;
+            if (File.Exists(option.GoogleStorageCredentialFile))
+            {
+                credential = await GoogleCredential.FromFileAsync(option.GoogleStorageCredentialFile, token);
+            }
+            else
+            {
+                var decodedJson = Encoding.UTF8.GetString(Convert.FromBase64String(option.GoogleStorageCredentialFile));
+                credential = GoogleCredential.FromJson(decodedJson);
+            }
+            return await StorageClient.CreateAsync(credential);
+        }
+    }
+
+    public class NoneRemoteServer : IObjectStorageRemoteServer
+    {
+        public Task<UploadResponse> UploadAsync(string targetPath, Stream data, CancellationToken token)
+        {
+            return Task.FromResult(new UploadResponse { IsSuccess = true, RemoteFilePath = targetPath });
+        }
+    }
+
+    public class AzureRemoteServer : IObjectStorageRemoteServer
+    {
+        private readonly BlobContainerClient _container;
+
+        public AzureRemoteServer(BlobContainerClient container)
+        {
+            _container = container;
+        }
+
+        public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, CancellationToken token)
+        {
+            var result = await _container.UploadBlobAsync(targetPath, data, token);
+            var response = result.GetRawResponse();
+            return new UploadResponse
+            {
+                IsSuccess = response.Status == 201,
+                ErrorMessage = response.ReasonPhrase,
+                RemoteFilePath = targetPath
+            };
+        }
+    }
+
+    public class GoogleRemoteServer : IObjectStorageRemoteServer
+    {
+        internal StorageClient Storage { get; }
+        internal string BucketName { get; }
+
+        public GoogleRemoteServer(StorageClient storage, string bucketName)
+        {
+            Storage = storage;
+            BucketName = bucketName;
+        }
+
+        public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, CancellationToken token)
+        {
+            try
+            {
+                Object uploaded = await Storage.UploadObjectAsync(BucketName, targetPath, null, data, cancellationToken: token);
+            }
+            catch (Exception e)
+            {
+                return new UploadResponse
+                {
+                    IsSuccess = false,
+                    RemoteFilePath = targetPath,
+                    ErrorMessage = e.Message
+                };
+            }
+            return new UploadResponse
+            {
+                IsSuccess = true, 
+                RemoteFilePath = targetPath
+            };
+        }
+    }
+
+    public class UploadResponse
+    {
+        public string RemoteFilePath { get; set; }
+        public bool IsSuccess { get; set; }
+        public string ErrorMessage { get; set; }
+    }
+}

TCC.Lib/Options/CompressOption.cs

@@ -19,8 +19,9 @@ public class CompressOption : TccOption
         public string AzBlobUrl { get; set; }
         public string AzBlobContainer { get; set; }
         public string AzSaS { get; set; }
-        public int? AzMbps { get; set; }
         public int? AzThread { get; set; }
+        public string GoogleStorageBucketName { get; set; }
+        public string GoogleStorageCredentialFile { get; set; }
         public UploadMode? UploadMode { get; set; }
     }
 }

TCC.Lib/TCC.Lib.csproj

@@ -7,6 +7,7 @@
 
   <ItemGroup>
     <PackageReference Include="Azure.Storage.Blobs" Version="12.8.1" />
+    <PackageReference Include="Google.Cloud.Storage.V1" Version="3.5.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="5.0.4">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>