diff --git a/TCC.Lib/ObjectStorage/IObjectRemoteServer.cs b/TCC.Lib/ObjectStorage/IObjectRemoteServer.cs deleted file mode 100644 index a5a08e7..0000000 --- a/TCC.Lib/ObjectStorage/IObjectRemoteServer.cs +++ /dev/null @@ -1,134 +0,0 @@ -using Azure.Storage.Blobs; -using Google.Apis.Auth.OAuth2; -using Google.Cloud.Storage.V1; -using System; -using System.IO; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using TCC.Lib.Dependencies; -using TCC.Lib.Options; -using Object = Google.Apis.Storage.v1.Data.Object; - -namespace TCC.Lib.ObjectStorage -{ - public interface IObjectStorageRemoteServer - { - Task UploadAsync(string targetPath, Stream data, CancellationToken token); - - public async Task UploadAsync(FileInfo file, DirectoryInfo rootFolder, CancellationToken token) - { - string targetPath = file.GetRelativeTargetPathTo(rootFolder); - await using FileStream uploadFileStream = File.OpenRead(file.FullName); - return await UploadAsync(targetPath, uploadFileStream, token); - } - } - - public static class RemoteServerFactory - { - public static async Task CreateRemoteServerAsync(this CompressOption option, CancellationToken token) - { - switch (option.UploadMode) - { - case UploadMode.AzureSdk: - var client = new BlobServiceClient(new Uri(option.AzBlobUrl + "/" + option.AzBlobContainer + "?" + option.AzSaS)); - BlobContainerClient container = client.GetBlobContainerClient(option.AzBlobContainer); - return new AzureRemoteServer(container); - case UploadMode.GoogleCloudStorage: - StorageClient storage = await GetGoogleStorageClient(option, token); - return new GoogleRemoteServer(storage, option.GoogleStorageBucketName); - case UploadMode.None: - case null: - return new NoneRemoteServer(); - default: - throw new ArgumentOutOfRangeException(); - } - } - - private static async Task GetGoogleStorageClient(CompressOption option, CancellationToken token) - { - GoogleCredential credential; - if (File.Exists(option.GoogleStorageCredentialFile)) - { - credential = await GoogleCredential.FromFileAsync(option.GoogleStorageCredentialFile, token); - } - else - { - var decodedJson = Encoding.UTF8.GetString(Convert.FromBase64String(option.GoogleStorageCredentialFile)); - credential = GoogleCredential.FromJson(decodedJson); - } - return await StorageClient.CreateAsync(credential); - } - } - - public class NoneRemoteServer : IObjectStorageRemoteServer - { - public Task UploadAsync(string targetPath, Stream data, CancellationToken token) - { - return Task.FromResult(new UploadResponse { IsSuccess = true, RemoteFilePath = targetPath }); - } - } - - public class AzureRemoteServer : IObjectStorageRemoteServer - { - private readonly BlobContainerClient _container; - - public AzureRemoteServer(BlobContainerClient container) - { - _container = container; - } - - public async Task UploadAsync(string targetPath, Stream data, CancellationToken token) - { - var result = await _container.UploadBlobAsync(targetPath, data, token); - var response = result.GetRawResponse(); - return new UploadResponse - { - IsSuccess = response.Status == 201, - ErrorMessage = response.ReasonPhrase, - RemoteFilePath = targetPath - }; - } - } - - public class GoogleRemoteServer : IObjectStorageRemoteServer - { - internal StorageClient Storage { get; } - internal string BucketName { get; } - - public GoogleRemoteServer(StorageClient storage, string bucketName) - { - Storage = storage; - BucketName = bucketName; - } - - public async Task UploadAsync(string targetPath, Stream data, CancellationToken token) - { - try - { - Object uploaded = await Storage.UploadObjectAsync(BucketName, targetPath, null, data, cancellationToken: token); - } - catch (Exception e) - { - return new UploadResponse - { - IsSuccess = false, - RemoteFilePath = targetPath, - ErrorMessage = e.Message - }; - } - return new UploadResponse - { - IsSuccess = true, - RemoteFilePath = targetPath - }; - } - } - - public class UploadResponse - { - public string RemoteFilePath { get; set; } - public bool IsSuccess { get; set; } - public string ErrorMessage { get; set; } - } -} diff --git a/TCC.Lib/Storage/AzureRemoteStorage.cs b/TCC.Lib/Storage/AzureRemoteStorage.cs new file mode 100644 index 0000000..43287c7 --- /dev/null +++ b/TCC.Lib/Storage/AzureRemoteStorage.cs @@ -0,0 +1,29 @@ +using Azure.Storage.Blobs; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace TCC.Lib.Storage +{ + public class AzureRemoteStorage : IRemoteStorage + { + private readonly BlobContainerClient _container; + + public AzureRemoteStorage(BlobContainerClient container) + { + _container = container; + } + + public async Task UploadAsync(string targetPath, Stream data, CancellationToken token) + { + var result = await _container.UploadBlobAsync(targetPath, data, token); + var response = result.GetRawResponse(); + return new UploadResponse + { + IsSuccess = response.Status == 201, + ErrorMessage = response.ReasonPhrase, + RemoteFilePath = targetPath + }; + } + } +} \ No newline at end of file diff --git a/TCC.Lib/Storage/GoogleRemoteStorage.cs b/TCC.Lib/Storage/GoogleRemoteStorage.cs new file mode 100644 index 0000000..2b85cdc --- /dev/null +++ b/TCC.Lib/Storage/GoogleRemoteStorage.cs @@ -0,0 +1,43 @@ +using Google.Cloud.Storage.V1; +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Object = Google.Apis.Storage.v1.Data.Object; + +namespace TCC.Lib.Storage +{ + public class GoogleRemoteStorage : IRemoteStorage + { + internal StorageClient Storage { get; } + internal string BucketName { get; } + + public GoogleRemoteStorage(StorageClient storage, string bucketName) + { + Storage = storage; + BucketName = bucketName; + } + + public async Task UploadAsync(string targetPath, Stream data, CancellationToken token) + { + try + { + Object uploaded = await Storage.UploadObjectAsync(BucketName, targetPath, null, data, cancellationToken: token); + } + catch (Exception e) + { + return new UploadResponse + { + IsSuccess = false, + RemoteFilePath = targetPath, + ErrorMessage = e.Message + }; + } + return new UploadResponse + { + IsSuccess = true, + RemoteFilePath = targetPath + }; + } + } +} \ No newline at end of file diff --git a/TCC.Lib/Storage/IRemoteStorage.cs b/TCC.Lib/Storage/IRemoteStorage.cs new file mode 100644 index 0000000..273348e --- /dev/null +++ b/TCC.Lib/Storage/IRemoteStorage.cs @@ -0,0 +1,19 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using TCC.Lib.Dependencies; + +namespace TCC.Lib.Storage +{ + public interface IRemoteStorage + { + Task UploadAsync(string targetPath, Stream data, CancellationToken token); + + public async Task UploadAsync(FileInfo file, DirectoryInfo rootFolder, CancellationToken token) + { + string targetPath = file.GetRelativeTargetPathTo(rootFolder); + await using FileStream uploadFileStream = File.OpenRead(file.FullName); + return await UploadAsync(targetPath, uploadFileStream, token); + } + } +} \ No newline at end of file diff --git a/TCC.Lib/Storage/NoneRemoteStorage.cs b/TCC.Lib/Storage/NoneRemoteStorage.cs new file mode 100644 index 0000000..eb3e942 --- /dev/null +++ b/TCC.Lib/Storage/NoneRemoteStorage.cs @@ -0,0 +1,14 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace TCC.Lib.Storage +{ + public class NoneRemoteStorage : IRemoteStorage + { + public Task UploadAsync(string targetPath, Stream data, CancellationToken token) + { + return Task.FromResult(new UploadResponse { IsSuccess = true, RemoteFilePath = targetPath }); + } + } +} diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs new file mode 100644 index 0000000..6d51fcc --- /dev/null +++ b/TCC.Lib/Storage/RemoteStorageFactory.cs @@ -0,0 +1,67 @@ +using Azure.Storage.Blobs; +using Google.Apis.Auth.OAuth2; +using Google.Cloud.Storage.V1; +using Microsoft.Extensions.Logging; +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using TCC.Lib.Options; + +namespace TCC.Lib.Storage +{ + public static class RemoteStorageFactory + { + public static async Task GetRemoteStorageAsync(this CompressOption option, ILogger logger, CancellationToken token) + { + switch (option.UploadMode) + { + case UploadMode.AzureSdk: + { + if (string.IsNullOrEmpty(option.AzBlobUrl) + || string.IsNullOrEmpty(option.AzBlobContainer) + || string.IsNullOrEmpty(option.AzSaS)) + { + logger.LogCritical("Configuration error for azure blob upload"); + return new NoneRemoteStorage(); + } + var client = new BlobServiceClient(new Uri(option.AzBlobUrl + "/" + option.AzBlobContainer + "?" + option.AzSaS)); + BlobContainerClient container = client.GetBlobContainerClient(option.AzBlobContainer); + return new AzureRemoteStorage(container); + } + case UploadMode.GoogleCloudStorage: + { + if (string.IsNullOrEmpty(option.GoogleStorageCredentialFile) + || string.IsNullOrEmpty(option.GoogleStorageBucketName)) + { + logger.LogCritical("Configuration error for google storage upload"); + return new NoneRemoteStorage(); + } + StorageClient storage = await GetGoogleStorageClient(option, token); + return new GoogleRemoteStorage(storage, option.GoogleStorageBucketName); + } + case UploadMode.None: + case null: + return new NoneRemoteStorage(); + default: + throw new ArgumentOutOfRangeException(); + } + } + + private static async Task GetGoogleStorageClient(CompressOption option, CancellationToken token) + { + GoogleCredential credential; + if (File.Exists(option.GoogleStorageCredentialFile)) + { + credential = await GoogleCredential.FromFileAsync(option.GoogleStorageCredentialFile, token); + } + else + { + var decodedJson = Encoding.UTF8.GetString(Convert.FromBase64String(option.GoogleStorageCredentialFile)); + credential = GoogleCredential.FromJson(decodedJson); + } + return await StorageClient.CreateAsync(credential); + } + } +} \ No newline at end of file diff --git a/TCC.Lib/Storage/UploadResponse.cs b/TCC.Lib/Storage/UploadResponse.cs new file mode 100644 index 0000000..c87940a --- /dev/null +++ b/TCC.Lib/Storage/UploadResponse.cs @@ -0,0 +1,9 @@ +namespace TCC.Lib.Storage +{ + public class UploadResponse + { + public string RemoteFilePath { get; set; } + public bool IsSuccess { get; set; } + public string ErrorMessage { get; set; } + } +} \ No newline at end of file diff --git a/TCC.Lib/TarCompressCrypt.cs b/TCC.Lib/TarCompressCrypt.cs index 468ee1e..6a6f150 100644 --- a/TCC.Lib/TarCompressCrypt.cs +++ b/TCC.Lib/TarCompressCrypt.cs @@ -17,9 +17,9 @@ using TCC.Lib.Database; using TCC.Lib.Dependencies; using TCC.Lib.Helpers; -using TCC.Lib.ObjectStorage; using TCC.Lib.Options; using TCC.Lib.PrepareBlocks; +using TCC.Lib.Storage; namespace TCC.Lib { @@ -60,7 +60,7 @@ public async Task Compress(CompressOption option) _logger.LogInformation("Starting compression job"); var po = ParallelizeOption(option); - IObjectStorageRemoteServer uploader = await option.CreateRemoteServerAsync(_cancellationTokenSource.Token); + IRemoteStorage uploader = await option.GetRemoteStorageAsync(_logger, _cancellationTokenSource.Token); var operationBlocks = await buffer .AsAsyncStream(_cancellationTokenSource.Token) @@ -93,12 +93,9 @@ public async Task Compress(CompressOption option) return ops; } - private async Task UploadBlockInternal(IObjectStorageRemoteServer uploader, CompressOption option, OperationCompressionBlock block, CancellationToken token) + private async Task UploadBlockInternal(IRemoteStorage uploader, CompressOption option, OperationCompressionBlock block, CancellationToken token) { - if (string.IsNullOrEmpty(option.AzBlobUrl) - || string.IsNullOrEmpty(option.AzBlobContainer) - || string.IsNullOrEmpty(option.AzSaS) - || option.UploadMode == null) + if (uploader is NoneRemoteStorage) { return block; } diff --git a/TCC.Tests/Upload/BlobStorageTests.cs b/TCC.Tests/Upload/BlobStorageTests.cs index 90e8566..aaabf2a 100644 --- a/TCC.Tests/Upload/BlobStorageTests.cs +++ b/TCC.Tests/Upload/BlobStorageTests.cs @@ -2,20 +2,17 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using System; -using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using TCC.Lib; using TCC.Lib.Benchmark; -using TCC.Lib.Dependencies; -using TCC.Lib.ObjectStorage; using TCC.Lib.Options; +using TCC.Lib.Storage; using Xunit; -namespace TCC.Tests +namespace TCC.Tests.Upload { public class UploadToStorageTests : IClassFixture { @@ -39,7 +36,7 @@ public async Task AzureUploadTest() AzSaS = GetEnvVar("AZ_SAS_TOKEN") }; opt.UploadMode = UploadMode.AzureSdk; - var uploader = await opt.CreateRemoteServerAsync(CancellationToken.None); + var uploader = await opt.GetRemoteStorageAsync(NullLogger.Instance, CancellationToken.None); var ok = await uploader.UploadAsync(data.Files.First(), new DirectoryInfo(toCompressFolder), CancellationToken.None); @@ -59,13 +56,14 @@ public async Task GoogleUploadTest() }; opt.UploadMode = UploadMode.GoogleCloudStorage; - var uploader = await opt.CreateRemoteServerAsync(CancellationToken.None); + var uploader = await opt.GetRemoteStorageAsync(NullLogger.Instance, CancellationToken.None); var ok = await uploader.UploadAsync(data.Files.First(), new DirectoryInfo(toCompressFolder), CancellationToken.None); Assert.True(ok.IsSuccess); - var gs = uploader as GoogleRemoteServer; + var gs = uploader as GoogleRemoteStorage; + await gs.Storage.DeleteObjectAsync(gs.BucketName, ok.RemoteFilePath); }