-
Notifications
You must be signed in to change notification settings - Fork 677
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for Google buckets storage (#233)
- Loading branch information
1 parent
873b133
commit 71db5d5
Showing
10 changed files
with
212 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,44 @@ | ||
| # Running BaGet on the Google Cloud | ||
| # Running BaGet on Google Cloud | ||
|
|
||
| !!! warning | ||
| This page is a work in progress! | ||
|
|
||
| Sadly, BaGet does not support GCP today. We're open source and accept contributions! | ||
| We're open source and accept contributions! | ||
| [Fork us on GitHub](https://github.com/loic-sharma/BaGet). | ||
|
|
||
| For now, please refer to the [Azure documentation](azure). | ||
| ## Google Cloud Storage | ||
|
|
||
| Packages can be stored in [Google Cloud Storage](https://cloud.google.com/storage/). | ||
|
|
||
| ### Setup | ||
|
|
||
| Follow the instructions in [Using Cloud Storage](https://cloud.google.com/appengine/docs/flexible/dotnet/using-cloud-storage) to: | ||
|
|
||
| * Create a bucket | ||
| * Set up a service account and download credentials | ||
| * Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path to the JSON file you downloaded | ||
|
|
||
| ### Configuration | ||
|
|
||
| Configure BaGet to use GCS by updating the [`appsettings.json`](https://github.com/loic-sharma/BaGet/blob/master/src/BaGet/appsettings.json) file: | ||
|
|
||
| ```json | ||
| { | ||
| ... | ||
|
|
||
| "Storage": { | ||
| "Type": "GoogleCloud", | ||
| "BucketName": "your-gcs-bucket" | ||
| }, | ||
|
|
||
| ... | ||
| } | ||
| ``` | ||
|
|
||
| ## Google Cloud SQL | ||
|
|
||
| * TODO | ||
|
|
||
| ## Google AppEngine | ||
|
|
||
| * TODO |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,7 @@ public enum StorageType | |
| { | ||
| FileSystem = 0, | ||
| AzureBlobStorage = 1, | ||
| AwsS3 = 2 | ||
| AwsS3 = 2, | ||
| GoogleCloud = 3, | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFrameworks>netstandard2.0;net461</TargetFrameworks> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Google.Cloud.Storage.V1" Version="2.2.1" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\BaGet.Core\BaGet.Core.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| using System.ComponentModel.DataAnnotations; | ||
| using BaGet.Core.Configuration; | ||
|
|
||
| namespace BaGet.GCP.Configuration | ||
| { | ||
| public class GoogleCloudStorageOptions : StorageOptions | ||
| { | ||
| [Required] | ||
| public string BucketName { get; set; } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| using BaGet.Core.Services; | ||
| using BaGet.GCP.Services; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
|
|
||
| namespace BaGet.GCP.Extensions | ||
| { | ||
| public static class ServiceCollectionExtensions | ||
| { | ||
| public static IServiceCollection AddGoogleCloudStorageService(this IServiceCollection services) | ||
| { | ||
| services.AddTransient<GoogleCloudStorageService>(); | ||
| return services; | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| using System; | ||
| using System.IO; | ||
| using System.Linq; | ||
| using System.Net; | ||
| using System.Security.Cryptography; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using BaGet.Core.Services; | ||
| using BaGet.GCP.Configuration; | ||
| using Google; | ||
| using Google.Cloud.Storage.V1; | ||
| using Microsoft.Extensions.Options; | ||
|
|
||
| namespace BaGet.GCP.Services | ||
| { | ||
| public class GoogleCloudStorageService : IStorageService | ||
| { | ||
| private readonly string _bucketName; | ||
|
|
||
| public GoogleCloudStorageService(IOptionsSnapshot<GoogleCloudStorageOptions> options) | ||
| { | ||
| if (options == null) | ||
| throw new ArgumentNullException(nameof(options)); | ||
|
|
||
| _bucketName = options.Value.BucketName; | ||
| } | ||
|
|
||
| public async Task<Stream> GetAsync(string path, CancellationToken cancellationToken = default) | ||
| { | ||
| using (var storage = await StorageClient.CreateAsync()) | ||
| { | ||
| var stream = new MemoryStream(); | ||
| await storage.DownloadObjectAsync(_bucketName, CoercePath(path), stream, cancellationToken: cancellationToken); | ||
| stream.Position = 0; | ||
| return stream; | ||
| } | ||
| } | ||
|
|
||
| public Task<Uri> GetDownloadUriAsync(string path, CancellationToken cancellationToken = default) | ||
| { | ||
| // returns an Authenticated Browser Download URL: https://cloud.google.com/storage/docs/request-endpoints#cookieauth | ||
| return Task.FromResult(new Uri($"https://storage.googleapis.com/{_bucketName}/{CoercePath(path).TrimStart('/')}")); | ||
| } | ||
|
|
||
| public async Task<PutResult> PutAsync(string path, Stream content, string contentType, CancellationToken cancellationToken = default) | ||
| { | ||
| using (var storage = await StorageClient.CreateAsync()) | ||
| using (var seekableContent = new MemoryStream()) | ||
| { | ||
| await content.CopyToAsync(seekableContent, 65536, cancellationToken); | ||
| seekableContent.Position = 0; | ||
|
|
||
| var objectName = CoercePath(path); | ||
|
|
||
| try | ||
| { | ||
| // attempt to upload, succeeding only if the object doesn't exist | ||
| await storage.UploadObjectAsync(_bucketName, objectName, contentType, seekableContent, new UploadObjectOptions { IfGenerationMatch = 0 }, cancellationToken); | ||
| return PutResult.Success; | ||
| } | ||
| catch (GoogleApiException e) when (e.HttpStatusCode == HttpStatusCode.PreconditionFailed) | ||
| { | ||
| // the object already exists; get the hash of its content from its metadata | ||
| var existingObject = await storage.GetObjectAsync(_bucketName, objectName, cancellationToken: cancellationToken); | ||
| var existingHash = Convert.FromBase64String(existingObject.Md5Hash); | ||
|
|
||
| // hash the content that was uploaded | ||
| seekableContent.Position = 0; | ||
| byte[] contentHash; | ||
| using (var md5 = MD5.Create()) | ||
| contentHash = md5.ComputeHash(seekableContent); | ||
|
|
||
| // conflict if the two hashes are different | ||
| return existingHash.SequenceEqual(contentHash) ? PutResult.AlreadyExists : PutResult.Conflict; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) | ||
| { | ||
| using (var storage = await StorageClient.CreateAsync()) | ||
| { | ||
| try | ||
| { | ||
| var obj = await storage.GetObjectAsync(_bucketName, CoercePath(path), cancellationToken: cancellationToken); | ||
| await storage.DeleteObjectAsync(obj, cancellationToken: cancellationToken); | ||
| } | ||
| catch (GoogleApiException e) when (e.HttpStatusCode == HttpStatusCode.NotFound) | ||
| { | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static string CoercePath(string path) | ||
| { | ||
| // although Google Cloud Storage objects exist in a flat namespace, using forward slashes allows the objects to | ||
| // be exposed as nested subdirectories, e.g., when browsing via Google Cloud Console | ||
| return path.Replace('\\', '/'); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters