From 71db5d580e0173f57c38c235e689ab4e652f91fc Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 24 Mar 2019 13:27:57 -0700 Subject: [PATCH] Add support for Google buckets storage (#233) --- BaGet.sln | 7 ++ docs/cloud/gcp.md | 41 ++++++- src/BaGet.AWS/S3StorageService.cs | 7 +- .../Configuration/StorageOptions.cs | 3 +- src/BaGet.GCP/BaGet.GCP.csproj | 15 +++ .../GoogleCloudStorageOptions.cs | 11 ++ .../Extensions/ServiceCollectionExtensions.cs | 15 +++ .../Services/GoogleCloudStorageService.cs | 101 ++++++++++++++++++ src/BaGet/BaGet.csproj | 1 + .../IServiceCollectionExtensions.cs | 17 +++ 10 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 src/BaGet.GCP/BaGet.GCP.csproj create mode 100644 src/BaGet.GCP/Configuration/GoogleCloudStorageOptions.cs create mode 100644 src/BaGet.GCP/Extensions/ServiceCollectionExtensions.cs create mode 100644 src/BaGet.GCP/Services/GoogleCloudStorageService.cs diff --git a/BaGet.sln b/BaGet.sln index 7321e4264..776a488fa 100644 --- a/BaGet.sln +++ b/BaGet.sln @@ -40,6 +40,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BaGet.Database.MySql", "src EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaGet.Database.PostgreSql", "src\BaGet.Database.PostgreSql\BaGet.Database.PostgreSql.csproj", "{F48F201A-4DEE-4D5B-9C0B-59490FE942FA}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BaGet.GCP", "src\BaGet.GCP\BaGet.GCP.csproj", "{D7D60BA0-FF7F-4B37-815C-74D487C5176E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -102,6 +104,10 @@ Global {F48F201A-4DEE-4D5B-9C0B-59490FE942FA}.Debug|Any CPU.Build.0 = Debug|Any CPU {F48F201A-4DEE-4D5B-9C0B-59490FE942FA}.Release|Any CPU.ActiveCfg = Release|Any CPU {F48F201A-4DEE-4D5B-9C0B-59490FE942FA}.Release|Any CPU.Build.0 = Release|Any CPU + {D7D60BA0-FF7F-4B37-815C-74D487C5176E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D7D60BA0-FF7F-4B37-815C-74D487C5176E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D7D60BA0-FF7F-4B37-815C-74D487C5176E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D7D60BA0-FF7F-4B37-815C-74D487C5176E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -121,6 +127,7 @@ Global {4C513AFC-BA7B-471D-B8F6-268E7AD2074C} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC} {A4375529-E855-4D46-AA4F-B3FE630C3DE1} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC} {F48F201A-4DEE-4D5B-9C0B-59490FE942FA} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC} + {D7D60BA0-FF7F-4B37-815C-74D487C5176E} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {1423C027-2C90-417F-8629-2A4CF107C055} diff --git a/docs/cloud/gcp.md b/docs/cloud/gcp.md index 503249070..6a333ab93 100644 --- a/docs/cloud/gcp.md +++ b/docs/cloud/gcp.md @@ -1,9 +1,44 @@ -# Running BaGet on the Google Cloud +# Running BaGet on Google Cloud !!! warning This page is a work in progress! -Sadly, BaGet does not support GCP today. We're open source and accept contributions! +We're open source and accept contributions! [Fork us on GitHub](https://github.com/loic-sharma/BaGet). -For now, please refer to the [Azure documentation](azure). \ No newline at end of file +## Google Cloud Storage + +Packages can be stored in [Google Cloud Storage](https://cloud.google.com/storage/). + +### Setup + +Follow the instructions in [Using Cloud Storage](https://cloud.google.com/appengine/docs/flexible/dotnet/using-cloud-storage) to: + +* Create a bucket +* Set up a service account and download credentials +* Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path to the JSON file you downloaded + +### Configuration + +Configure BaGet to use GCS by updating the [`appsettings.json`](https://github.com/loic-sharma/BaGet/blob/master/src/BaGet/appsettings.json) file: + +```json +{ + ... + + "Storage": { + "Type": "GoogleCloud", + "BucketName": "your-gcs-bucket" + }, + + ... +} +``` + +## Google Cloud SQL + +* TODO + +## Google AppEngine + +* TODO diff --git a/src/BaGet.AWS/S3StorageService.cs b/src/BaGet.AWS/S3StorageService.cs index 056b2784a..c52d11374 100644 --- a/src/BaGet.AWS/S3StorageService.cs +++ b/src/BaGet.AWS/S3StorageService.cs @@ -17,11 +17,14 @@ public class S3StorageService : IStorageService private readonly string _prefix; private readonly AmazonS3Client _client; - public S3StorageService(IOptions options, AmazonS3Client client) + public S3StorageService(IOptionsSnapshot options, AmazonS3Client client) { + if (options == null) + throw new ArgumentNullException(nameof(options)); + _bucket = options.Value.Bucket; _prefix = options.Value.Prefix; - _client = client; + _client = client ?? throw new ArgumentNullException(nameof(client)); if (!string.IsNullOrEmpty(_prefix) && !_prefix.EndsWith(Separator)) _prefix += Separator; diff --git a/src/BaGet.Core/Configuration/StorageOptions.cs b/src/BaGet.Core/Configuration/StorageOptions.cs index 4915307e0..486394281 100644 --- a/src/BaGet.Core/Configuration/StorageOptions.cs +++ b/src/BaGet.Core/Configuration/StorageOptions.cs @@ -9,6 +9,7 @@ public enum StorageType { FileSystem = 0, AzureBlobStorage = 1, - AwsS3 = 2 + AwsS3 = 2, + GoogleCloud = 3, } } diff --git a/src/BaGet.GCP/BaGet.GCP.csproj b/src/BaGet.GCP/BaGet.GCP.csproj new file mode 100644 index 000000000..2744867cc --- /dev/null +++ b/src/BaGet.GCP/BaGet.GCP.csproj @@ -0,0 +1,15 @@ + + + + netstandard2.0;net461 + + + + + + + + + + + diff --git a/src/BaGet.GCP/Configuration/GoogleCloudStorageOptions.cs b/src/BaGet.GCP/Configuration/GoogleCloudStorageOptions.cs new file mode 100644 index 000000000..8adaf90eb --- /dev/null +++ b/src/BaGet.GCP/Configuration/GoogleCloudStorageOptions.cs @@ -0,0 +1,11 @@ +using System.ComponentModel.DataAnnotations; +using BaGet.Core.Configuration; + +namespace BaGet.GCP.Configuration +{ + public class GoogleCloudStorageOptions : StorageOptions + { + [Required] + public string BucketName { get; set; } + } +} diff --git a/src/BaGet.GCP/Extensions/ServiceCollectionExtensions.cs b/src/BaGet.GCP/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 000000000..b9f5773a6 --- /dev/null +++ b/src/BaGet.GCP/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,15 @@ +using BaGet.Core.Services; +using BaGet.GCP.Services; +using Microsoft.Extensions.DependencyInjection; + +namespace BaGet.GCP.Extensions +{ + public static class ServiceCollectionExtensions + { + public static IServiceCollection AddGoogleCloudStorageService(this IServiceCollection services) + { + services.AddTransient(); + return services; + } + } +} diff --git a/src/BaGet.GCP/Services/GoogleCloudStorageService.cs b/src/BaGet.GCP/Services/GoogleCloudStorageService.cs new file mode 100644 index 000000000..ce7beab40 --- /dev/null +++ b/src/BaGet.GCP/Services/GoogleCloudStorageService.cs @@ -0,0 +1,101 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using BaGet.Core.Services; +using BaGet.GCP.Configuration; +using Google; +using Google.Cloud.Storage.V1; +using Microsoft.Extensions.Options; + +namespace BaGet.GCP.Services +{ + public class GoogleCloudStorageService : IStorageService + { + private readonly string _bucketName; + + public GoogleCloudStorageService(IOptionsSnapshot options) + { + if (options == null) + throw new ArgumentNullException(nameof(options)); + + _bucketName = options.Value.BucketName; + } + + public async Task GetAsync(string path, CancellationToken cancellationToken = default) + { + using (var storage = await StorageClient.CreateAsync()) + { + var stream = new MemoryStream(); + await storage.DownloadObjectAsync(_bucketName, CoercePath(path), stream, cancellationToken: cancellationToken); + stream.Position = 0; + return stream; + } + } + + public Task GetDownloadUriAsync(string path, CancellationToken cancellationToken = default) + { + // returns an Authenticated Browser Download URL: https://cloud.google.com/storage/docs/request-endpoints#cookieauth + return Task.FromResult(new Uri($"https://storage.googleapis.com/{_bucketName}/{CoercePath(path).TrimStart('/')}")); + } + + public async Task PutAsync(string path, Stream content, string contentType, CancellationToken cancellationToken = default) + { + using (var storage = await StorageClient.CreateAsync()) + using (var seekableContent = new MemoryStream()) + { + await content.CopyToAsync(seekableContent, 65536, cancellationToken); + seekableContent.Position = 0; + + var objectName = CoercePath(path); + + try + { + // attempt to upload, succeeding only if the object doesn't exist + await storage.UploadObjectAsync(_bucketName, objectName, contentType, seekableContent, new UploadObjectOptions { IfGenerationMatch = 0 }, cancellationToken); + return PutResult.Success; + } + catch (GoogleApiException e) when (e.HttpStatusCode == HttpStatusCode.PreconditionFailed) + { + // the object already exists; get the hash of its content from its metadata + var existingObject = await storage.GetObjectAsync(_bucketName, objectName, cancellationToken: cancellationToken); + var existingHash = Convert.FromBase64String(existingObject.Md5Hash); + + // hash the content that was uploaded + seekableContent.Position = 0; + byte[] contentHash; + using (var md5 = MD5.Create()) + contentHash = md5.ComputeHash(seekableContent); + + // conflict if the two hashes are different + return existingHash.SequenceEqual(contentHash) ? PutResult.AlreadyExists : PutResult.Conflict; + } + } + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + using (var storage = await StorageClient.CreateAsync()) + { + try + { + var obj = await storage.GetObjectAsync(_bucketName, CoercePath(path), cancellationToken: cancellationToken); + await storage.DeleteObjectAsync(obj, cancellationToken: cancellationToken); + } + catch (GoogleApiException e) when (e.HttpStatusCode == HttpStatusCode.NotFound) + { + } + } + } + + private static string CoercePath(string path) + { + // although Google Cloud Storage objects exist in a flat namespace, using forward slashes allows the objects to + // be exposed as nested subdirectories, e.g., when browsing via Google Cloud Console + return path.Replace('\\', '/'); + } + } +} diff --git a/src/BaGet/BaGet.csproj b/src/BaGet/BaGet.csproj index b98c37353..8d5e35b1e 100644 --- a/src/BaGet/BaGet.csproj +++ b/src/BaGet/BaGet.csproj @@ -31,6 +31,7 @@ + diff --git a/src/BaGet/Extensions/IServiceCollectionExtensions.cs b/src/BaGet/Extensions/IServiceCollectionExtensions.cs index ad4860b6c..14b4154b3 100644 --- a/src/BaGet/Extensions/IServiceCollectionExtensions.cs +++ b/src/BaGet/Extensions/IServiceCollectionExtensions.cs @@ -18,6 +18,9 @@ using BaGet.Database.PostgreSql; using BaGet.Database.Sqlite; using BaGet.Database.SqlServer; +using BaGet.GCP.Configuration; +using BaGet.GCP.Extensions; +using BaGet.GCP.Services; using BaGet.Protocol; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; @@ -44,6 +47,7 @@ public static IServiceCollection ConfigureBaGet( services.ConfigureAzure(configuration); services.ConfigureAws(configuration); + services.ConfigureGcp(configuration); if (httpServices) { @@ -144,6 +148,15 @@ public static IServiceCollection ConfigureAws( return services; } + public static IServiceCollection ConfigureGcp( + this IServiceCollection services, + IConfiguration configuration) + { + services.ConfigureAndValidate(configuration.GetSection(nameof(BaGetOptions.Storage))); + + return services; + } + public static IServiceCollection AddStorageProviders(this IServiceCollection services) { services.AddTransient(); @@ -152,6 +165,7 @@ public static IServiceCollection AddStorageProviders(this IServiceCollection ser services.AddBlobStorageService(); services.AddS3StorageService(); + services.AddGoogleCloudStorageService(); services.AddTransient(provider => { @@ -168,6 +182,9 @@ public static IServiceCollection AddStorageProviders(this IServiceCollection ser case StorageType.AwsS3: return provider.GetRequiredService(); + case StorageType.GoogleCloud: + return provider.GetRequiredService(); + default: throw new InvalidOperationException( $"Unsupported storage service: {options.Value.Storage.Type}");