diff --git a/src/Monai.Deploy.Storage.sln b/src/Monai.Deploy.Storage.sln index 6344b5e..a081391 100644 --- a/src/Monai.Deploy.Storage.sln +++ b/src/Monai.Deploy.Storage.sln @@ -23,6 +23,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Monai.Deploy.Storage.MinIO" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Monai.Deploy.Storage.MinIO.Tests", "Plugins\MinIO\Tests\Monai.Deploy.Storage.MinIO.Tests.csproj", "{FCB3FCA4-2BB6-4921-9715-CDECF343C6E2}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monai.Deploy.Storage.AzureBlob", "Plugins\AzureBlob\Monai.Deploy.Storage.AzureBlob\Monai.Deploy.Storage.AzureBlob.csproj", "{054DC580-A3ED-48AE-9706-E0CFDE6C171F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monai.Deploy.Storage.AzureBlob.Tests", "Plugins\AzureBlob\Monai.Deploy.Storage.AzureBlob.Tests\Monai.Deploy.Storage.AzureBlob.Tests.csproj", "{FDADE27C-70DC-401D-B058-041D4A587548}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -57,6 +61,14 @@ Global {FCB3FCA4-2BB6-4921-9715-CDECF343C6E2}.Debug|Any CPU.Build.0 = Debug|Any CPU {FCB3FCA4-2BB6-4921-9715-CDECF343C6E2}.Release|Any CPU.ActiveCfg = Release|Any CPU {FCB3FCA4-2BB6-4921-9715-CDECF343C6E2}.Release|Any CPU.Build.0 = Release|Any CPU + {054DC580-A3ED-48AE-9706-E0CFDE6C171F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {054DC580-A3ED-48AE-9706-E0CFDE6C171F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {054DC580-A3ED-48AE-9706-E0CFDE6C171F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {054DC580-A3ED-48AE-9706-E0CFDE6C171F}.Release|Any CPU.Build.0 = Release|Any CPU + {FDADE27C-70DC-401D-B058-041D4A587548}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FDADE27C-70DC-401D-B058-041D4A587548}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FDADE27C-70DC-401D-B058-041D4A587548}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FDADE27C-70DC-401D-B058-041D4A587548}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -65,6 +77,8 @@ Global {9E605292-D0F4-4E56-B723-D98397E07A77} = {0F380AAC-016C-4B0E-808E-059F6F17EB29} {0292D249-4FDD-4C4A-9D81-669E4375D23A} = {0F380AAC-016C-4B0E-808E-059F6F17EB29} {FCB3FCA4-2BB6-4921-9715-CDECF343C6E2} = {0F380AAC-016C-4B0E-808E-059F6F17EB29} + {054DC580-A3ED-48AE-9706-E0CFDE6C171F} = {0F380AAC-016C-4B0E-808E-059F6F17EB29} + {FDADE27C-70DC-401D-B058-041D4A587548} = {0F380AAC-016C-4B0E-808E-059F6F17EB29} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E1105263-9CBF-45AA-BAC3-BD8504C1B962} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/AssemblyInfo.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/AssemblyInfo.cs new file mode 100644 index 0000000..62c06f4 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/AssemblyInfo.cs @@ -0,0 +1,23 @@ +/* + * Copyright 2022 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Xunit; +//Optional +[assembly: CollectionBehavior(DisableTestParallelization = true)] +//Optional +[assembly: TestCaseOrderer("Xunit.Extensions.Ordering.TestCaseOrderer", "Xunit.Extensions.Ordering")] +//Optional +[assembly: TestCollectionOrderer("Xunit.Extensions.Ordering.CollectionOrderer", "Xunit.Extensions.Ordering")] diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Class1.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Class1.cs new file mode 100644 index 0000000..d802207 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Class1.cs @@ -0,0 +1,7 @@ +namespace Monai.Deploy.Storage.AzureBlob.Tests +{ + public class Class1 + { + + } +} \ No newline at end of file diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureBlobServiceTests.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureBlobServiceTests.cs new file mode 100644 index 0000000..2b89c04 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureBlobServiceTests.cs @@ -0,0 +1,200 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; +using Xunit.Extensions.Ordering; + +namespace Monai.Deploy.Storage.AzureBlob.Tests.Integration +{ + [Order(0), Collection("AzureBlobStorage")] + public class AzureBlobServiceTests + { + private readonly Mock> _logger; + private readonly AzureBlobStorageFixture _fixture; + private readonly AzureBlobStorageService _azureBlobService; + private readonly string _testFileName; + private readonly string _testFileNameCopy; + + public AzureBlobServiceTests(AzureBlobStorageFixture fixture) + { + _logger = new Mock>(); + _fixture = fixture ?? throw new ArgumentNullException(nameof(fixture)); + _azureBlobService = new AzureBlobStorageService(_fixture.ClientFactory, _fixture.Configurations, _logger.Object); + _testFileName = $"Tao-Te-Ching/Laozi/chapter-one.zip"; + _testFileNameCopy = $"Tao-Te-Ching/Laozi/chapter-one=backup.zip"; + } + + + [Fact, Order(1)] + public async Task S01_GivenABucketToAzureBlob() + { + var exception = await Record.ExceptionAsync(async () => + { + //await _azureBlobService.CreateFolderAsync(_fixture.ContainerName, ""); + var containerClient = _fixture.ClientFactory.GetBlobContainerClient(_fixture.ContainerName); + await containerClient.CreateIfNotExistsAsync().ConfigureAwait(false); + }).ConfigureAwait(false); + + Assert.Null(exception); + } + + [Fact, Order(2)] + public async Task S02_GivenASetOfDataAvailableToAzureBlob() + { + var exception = await Record.ExceptionAsync(async () => + { + await _fixture.GenerateAndUploadData().ConfigureAwait(false); + }).ConfigureAwait(false); + + Assert.Null(exception); + } + + [Theory, Order(3)] + [InlineData(null, 4)] + [InlineData("dir-1/", 1)] + [InlineData("dir-2/", 2)] + public async Task S03_WhenListObjectsAsyncIsCalled_ExpectItToListObjectsBasedOnParameters(string? prefix, int count) + { + var actual = await _azureBlobService.ListObjectsAsync(_fixture.ContainerName, prefix, true).ConfigureAwait(false); + + actual.Should().NotBeEmpty() + .And.HaveCount(count); + + var expected = _fixture.Files.ToList(); + if (prefix is not null) + { + expected = expected.Where(p => p.StartsWith(prefix)).ToList(); + } + actual.Select(p => p.FilePath).Should().BeEquivalentTo(expected); + } + + [Fact, Order(4)] + public async Task S04_WhenVerifyObjectsExistAsyncIsCalled_ExpectToReturnAll() + { + var actual = await _azureBlobService.VerifyObjectsExistAsync(_fixture.ContainerName, _fixture.Files).ConfigureAwait(false); + + actual.Should().NotBeEmpty() + .And.HaveCount(_fixture.Files.Count); + + actual.Should().ContainValues(true); + } + + [Fact, Order(5)] + public async Task S05_GivenAFileUploadedToAzureBlob() + { + var data = _fixture.GetRandomBytes(); + var stream = new MemoryStream(data); + await _azureBlobService.PutObjectAsync(_fixture.ContainerName, _testFileName, stream, data.Length, "application/binary", null).ConfigureAwait(false); + + var callback = (Stream stream) => + { + var actual = new MemoryStream(); + stream.CopyTo(actual); + actual.ToArray().Should().Equal(data); + }; + var client = _fixture.ClientFactory.GetBlobClient(_fixture.ContainerName, _testFileName); + + var fileContentsStream = new MemoryStream(); + await client.DownloadToAsync(fileContentsStream).ConfigureAwait(false); + fileContentsStream.ToArray().Should().Equal(data); + + var prop = await client.GetPropertiesAsync().ConfigureAwait(false); + prop.Value.ContentLength.Should().Be(data.Length); + } + + [Fact, Order(6)] + public async Task S06_ExpectTheFileToBeBeDownloadable() + { + var stream = await _azureBlobService.GetObjectAsync(_fixture.ContainerName, _testFileName).ConfigureAwait(false); + Assert.NotNull(stream); + var ms = new MemoryStream(); + stream.CopyTo(ms); + var data = ms.ToArray(); + + var original = await DownloadData(_testFileName).ConfigureAwait(false); + + Assert.NotNull(original); + + data.Should().Equal(original); + } + + [Fact, Order(7)] + public async Task S07_GivenACopyOfTheFile() + { + await _azureBlobService.CopyObjectAsync(_fixture.ContainerName, _testFileName, _fixture.ContainerName, _testFileNameCopy).ConfigureAwait(false); + + var original = await DownloadData(_testFileName).ConfigureAwait(false); + var copy = await DownloadData(_testFileNameCopy).ConfigureAwait(false); + + Assert.NotNull(original); + Assert.NotNull(copy); + + copy.Should().Equal(original); + } + + [Fact, Order(8)] + public async Task S08_ExpectedBothOriginalAndCopiedToExist() + { + var files = new List() { _testFileName, _testFileNameCopy, "file-does-not-exist" }; + var expectedResults = new List() { true, true, false }; + var results = await _azureBlobService.VerifyObjectsExistAsync(_fixture.ContainerName, files).ConfigureAwait(false); + + Assert.NotNull(results); + + results.Should().ContainKeys(files); + results.Should().ContainValues(expectedResults); + + for (var i = 0; i < files.Count; i++) + { + var file = files[i]; + var result = await _azureBlobService.VerifyObjectExistsAsync(_fixture.ContainerName, file).ConfigureAwait(false); + Assert.Equal(expectedResults[i], result); + } + } + + [Fact, Order(9)] + public async Task S09_GivenADirectoryCreatedToAzureBlob() + { + var folderName = "my-folder"; + await _azureBlobService.CreateFolderAsync(_fixture.ContainerName, folderName).ConfigureAwait(false); + var result = await _azureBlobService.VerifyObjectExistsAsync(_fixture.ContainerName, $"{folderName}/stubFile.txt").ConfigureAwait(false); + + Assert.True(result); + } + + [Fact, Order(10)] + public async Task S10_ExpectTheDirectoryToBeRemovable() + { + var folderName = "my - folder / stubFile.txt"; + await _azureBlobService.RemoveObjectAsync(_fixture.ContainerName, folderName).ConfigureAwait(false); + var result = await _azureBlobService.VerifyObjectExistsAsync(_fixture.ContainerName, $"{folderName}/stubFile.txt").ConfigureAwait(false); + Assert.False(result); + + var files = new List() { _testFileName, _testFileNameCopy, "file-does-not-exist" }; + await _azureBlobService.RemoveObjectsAsync(_fixture.ContainerName, files).ConfigureAwait(false); + } + + [Fact, Order(11)] + public async Task S11_ExpectTheFilesToBeRemovable() + { + var files = new List() { _testFileName, _testFileNameCopy, "file-does-not-exist" }; + await _azureBlobService.RemoveObjectsAsync(_fixture.ContainerName, files).ConfigureAwait(false); + + for (var i = 0; i < files.Count; i++) + { + var file = files[i]; + var result = await _azureBlobService.VerifyObjectExistsAsync(_fixture.ContainerName, file).ConfigureAwait(false); + Assert.False(result); + } + } + + private async Task DownloadData(string filename) + { + var copiedStream = new MemoryStream(); + + var client = _fixture.ClientFactory.GetBlobClient(_fixture.ContainerName, filename); + await client.DownloadToAsync(copiedStream).ConfigureAwait(false); + return copiedStream.ToArray(); + } + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureBlobStorageFixture.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureBlobStorageFixture.cs new file mode 100644 index 0000000..e1bbb62 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureBlobStorageFixture.cs @@ -0,0 +1,107 @@ +/* + * Copyright 2022 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Storage.Configuration; +using Moq; +using Xunit; + +namespace Monai.Deploy.Storage.AzureBlob.Tests.Integration +{ + [CollectionDefinition("AzureBlobStorage")] + public class AzureBlobStorageCollection : ICollectionFixture + { + // This class has no code, and is never created. Its purpose is simply + // to be the place to apply [CollectionDefinition] and all the + // ICollectionFixture<> interfaces. + } + + public class AzureBlobStorageFixture : IAsyncDisposable + { + const int MaxFileSize = 104857600; + private readonly Random _random; + private readonly List _files; + + public IOptions Configurations { get; } + public AzureBlobClientFactory ClientFactory { get; } + public IReadOnlyList Files { get => _files; } + public string ContainerName { get; } + + public AzureBlobStorageFixture() + { + _random = new Random(); + _files = new List(); + + Configurations = Options.Create(new StorageServiceConfiguration()); + Configurations.Value.Settings.Add(ConfigurationKeys.ConnectionString, "UseDevelopmentStorage=true"); + + ClientFactory = new AzureBlobClientFactory(Configurations, new Mock>().Object); + ContainerName = $"md-test-{_random.Next(1000)}"; + } + + internal async Task GenerateAndUploadData() + { + await GenerateAndUploadFile($"{Guid.NewGuid()}").ConfigureAwait(false); + await GenerateAndUploadFile($"dir-1/{Guid.NewGuid()}").ConfigureAwait(false); + await GenerateAndUploadFile($"dir-2/{Guid.NewGuid()}").ConfigureAwait(false); + await GenerateAndUploadFile($"dir-2/a/b/{Guid.NewGuid()}").ConfigureAwait(false); + } + + private async Task GenerateAndUploadFile(string filePath) + { + var data = GetRandomBytes(); + var stream = new MemoryStream(data); + var client = ClientFactory.GetBlobClient(ContainerName, filePath); + await client.UploadAsync(stream).ConfigureAwait(false); + _files.Add(filePath); + } + + public byte[] GetRandomBytes() + { + return new byte[_random.Next(1, MaxFileSize)]; + } + + public async ValueTask DisposeAsync() + { + await RemoveData().ConfigureAwait(false); + await RemoveBucket().ConfigureAwait(false); + } + + private async Task RemoveBucket() + { + var client = ClientFactory.GetBlobContainerClient(ContainerName); + var exists = await client.ExistsAsync().ConfigureAwait(false); + if (exists) + { + var resultSegment = client.GetBlobsAsync(prefix: ContainerName).AsPages(default, 100); + + await foreach (var blobPage in resultSegment) + { + foreach (var blobItem in blobPage.Values) + { + await ClientFactory.GetBlobClient(ContainerName, blobItem.Name).DeleteAsync().ConfigureAwait(false); + }; + } + } + } + + private async Task RemoveData() + { + await RemoveBucket().ConfigureAwait(false); + } + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureHealthCheckTest.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureHealthCheckTest.cs new file mode 100644 index 0000000..72900a2 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Integration/AzureHealthCheckTest.cs @@ -0,0 +1,51 @@ +/* + * Copyright 2022 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Storage.AzureBlob; +using Monai.Deploy.Storage.Configuration; +using Moq; +using Xunit; +using Xunit.Extensions.Ordering; + +namespace Monai.Deploy.Storage.AzureBlob.Tests.Integration +{ + [Order(10)] + public class AzureHealthCheckTest + { + private readonly AzureBlobClientFactory _azureBlobClientFactory; + private readonly Mock> _logger = new Mock>(); + + public AzureHealthCheckTest() + { + var ops = new StorageServiceConfiguration { Settings = new Dictionary { { "azureBlobConnectionString", "UseDevelopmentStorage=true" } } }; + var options = Options.Create(ops); + _azureBlobClientFactory = new AzureBlobClientFactory(options, new Mock>().Object); + } + + [Fact] + public async Task CheckHealthAsync_WhenListBucketSucceeds_ReturnHealthy() + { + var healthCheck = new AzureBlobHealthCheck(_azureBlobClientFactory, new Mock>().Object); + var results = await healthCheck.CheckHealthAsync(new HealthCheckContext()).ConfigureAwait(false); + + Assert.Equal(HealthStatus.Healthy, results.Status); + Assert.Null(results.Exception); + } + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Monai.Deploy.Storage.AzureBlob.Tests.csproj b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Monai.Deploy.Storage.AzureBlob.Tests.csproj new file mode 100644 index 0000000..8e7125a --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Monai.Deploy.Storage.AzureBlob.Tests.csproj @@ -0,0 +1,32 @@ + + + + net6.0 + enable + enable + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Unit/AzureBlobClientFactoryTests.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Unit/AzureBlobClientFactoryTests.cs new file mode 100644 index 0000000..5e64a3c --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Unit/AzureBlobClientFactoryTests.cs @@ -0,0 +1,41 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Storage.Configuration; +using Moq; +using Xunit; + +namespace Monai.Deploy.Storage.AzureBlob.Tests.Unit +{ + public class AzureBlobClientFactoryTests + { + private readonly Mock> _logger; + + public AzureBlobClientFactoryTests() + { + _logger = new Mock>(); + } + + [Fact] + public void ShouldThrowOnNullOptions() + { + var factoryMock = new Mock(); + Assert.Throws(() => new AzureBlobClientFactory(null, _logger.Object)); + } + + [Fact] + public void ShouldThrowOnNullLogger() + { + var factoryMock = new Mock(); + Assert.Throws(() => new AzureBlobClientFactory(Options.Create(new StorageServiceConfiguration()), null)); + } + + + [Fact] + public void ShouldThrowOnMisssingOptionsKey() + { + var factoryMock = new Mock(); + var options = Options.Create(new StorageServiceConfiguration { Settings = new Dictionary { { "endpoint", "somthing" } } }); + Assert.Throws(() => new AzureBlobClientFactory(options, _logger.Object)); + } + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Unit/AzureBlobHealthCheckTest.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Unit/AzureBlobHealthCheckTest.cs new file mode 100644 index 0000000..2c474a9 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Unit/AzureBlobHealthCheckTest.cs @@ -0,0 +1,48 @@ +/* + * Copyright 2022-2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace Monai.Deploy.Storage.AzureBlob.Tests +{ + public class AzureBlobHealthCheckTest + { + private readonly Mock _azureblobClientFactory; + private readonly Mock> _logger; + + public AzureBlobHealthCheckTest() + { + _azureblobClientFactory = new Mock(); + _logger = new Mock>(); + } + + [Fact] + public async Task CheckHealthAsync_WhenFailedToListBucket_ReturnUnhealthy() + { + _azureblobClientFactory.Setup(p => p.GetBlobContainerClient(It.IsAny())).Throws(new Exception("error")); + + var healthCheck = new AzureBlobHealthCheck(_azureblobClientFactory.Object, _logger.Object); + var results = await healthCheck.CheckHealthAsync(new HealthCheckContext()).ConfigureAwait(false); + + Assert.Equal(HealthStatus.Unhealthy, results.Status); + Assert.NotNull(results.Exception); + Assert.Equal("error", results.Exception.Message); + } + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Unit/AzureBlobServiceTests.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Unit/AzureBlobServiceTests.cs new file mode 100644 index 0000000..5dd19f2 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/Unit/AzureBlobServiceTests.cs @@ -0,0 +1,85 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Storage.Configuration; +using Moq; +using Xunit; + +namespace Monai.Deploy.Storage.AzureBlob.Tests.Unit +{ + public class AzureBlobServiceTests + { + private readonly Mock> _logger; + + public AzureBlobServiceTests() + { + _logger = new Mock>(); + } + + [Fact] + public async Task ShouldThrowOnNullOptions() + { + var factoryMock = new Mock(); + Assert.Throws(() => new AzureBlobStorageService(factoryMock.Object, null, new Mock>().Object)); + } + + [Fact] + public async Task ShouldThrowOnNullFactory() + { + var options = Options.Create(new StorageServiceConfiguration { Settings = new Dictionary { { "endpoint", "somthing" } } }); + Assert.Throws(() => new AzureBlobStorageService(null, options, new Mock>().Object)); + } + + [Fact] + public async Task ShouldThrowOnNullLogger() + { + var options = Options.Create(new StorageServiceConfiguration { Settings = new Dictionary { { "endpoint", "somthing" } } }); + var factoryMock = new Mock(); + Assert.Throws(() => new AzureBlobStorageService(factoryMock.Object, options, null)); + } + + [Fact] + public async Task ShouldThrowOnMissingOptionsKey() + { + var options = Options.Create(new StorageServiceConfiguration { Settings = new Dictionary { { "endpoint", "somthing" } } }); + var factoryMock = new Mock(); + Assert.Throws(() => new AzureBlobStorageService(factoryMock.Object, options, new Mock>().Object)); + } + + [Fact] + public async Task ShouldListBlobs() + { + var ops = new StorageServiceConfiguration { Settings = new Dictionary { { "azureBlobConnectionString", "UseDevelopmentStorage=true" } } }; + var options = Options.Create(ops); + var factory = new AzureBlobClientFactory(options, new Mock>().Object); + var service = new AzureBlobStorageService(factory, options, new Mock>().Object); + + var result = await service.ListObjectsAsync("basecontainer", "Aw", false); + Assert.Single(result); + + result = await service.ListObjectsAsync("basecontainer", "no", true); + Assert.Equal(2, result.Count); + + var stream = await service.GetObjectAsync("basecontainer", "other2/noretain.json"); + //await service.CopyObjectAsync("basecontainer/other", "folder/noretain.json", "basecontainer", "other2/noretain.json"); + var exists = await service.VerifyObjectExistsAsync("basecontainer/other2", "noretain.json"); + var exists2 = await service.VerifyObjectsExistAsync("basecontainer", new List { "other2/noretain.json", "other/noretain.json" }); + + var localFilePath = "C:\\Users\\NeilSouth\\source\\repos\\monai-deploy-storage\\src\\Plugins\\AzureBlob\\Monai.Deploy.Storage.AzureBlob.Tests\\Unit\\AzureBlobServiceTests.cs"; + var fileStream = File.OpenRead(localFilePath); + await service.PutObjectAsync( + "basecontainer/other", + "newFile.cs", + fileStream, + fileStream.Length, + "text/plain", + new Dictionary { { "author", "neil" } }); + + //await service.RemoveObjectAsync("basecontainer", "newFile.cs"); + + await service.RemoveObjectsAsync("basecontainer", new List { "other/newFile.cs", "other/folder/noretain.json" }); + + await service.CreateFolderAsync("basecontainer", "my/new/folder"); + + } + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/docker-compose.yml b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/docker-compose.yml new file mode 100644 index 0000000..81e0ebd --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob.Tests/docker-compose.yml @@ -0,0 +1,22 @@ +# Copyright 2022 MONAI Consortium +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.9" +services: + azurite: + image: "mcr.microsoft.com/azure-storage/azurite" + ports: + - 10000:10000 + - 10001:10001 + - 10002:10002 diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobClientFactory.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobClientFactory.cs new file mode 100644 index 0000000..18ba762 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobClientFactory.cs @@ -0,0 +1,72 @@ +using Ardalis.GuardClauses; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Specialized; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Storage.Configuration; + +namespace Monai.Deploy.Storage.AzureBlob +{ + public class AzureBlobClientFactory : IAzureBlobClientFactory + { + private readonly ILogger _logger; + private readonly IOptions _options; + private StorageServiceConfiguration Options { get; } + + private readonly BlobServiceClient _blobServiceClient; + + public AzureBlobClientFactory(IOptions options, ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + + var configuration = options.Value; + ValidateConfiguration(configuration); + Options = configuration; + + _blobServiceClient = new BlobServiceClient(Options.Settings[ConfigurationKeys.ConnectionString]); + } + + public BlobClient GetBlobClient(BlobContainerClient containerClient, string blob) + { + return containerClient.GetBlobClient(blob); + } + + public BlobClient GetBlobClient(string containerName, string blob) + { + return GetBlobContainerClient(containerName).GetBlobClient(blob); + } + + public BlockBlobClient GetBlobBlockClient(BlobContainerClient containerClient, string blob) + { + return containerClient.GetBlockBlobClient(blob); + } + + public BlockBlobClient GetBlobBlockClient(string containerName, string blob) + { + return GetBlobContainerClient(containerName).GetBlockBlobClient(blob); + } + + public BlobContainerClient GetBlobContainerClient(string containerName) + { + return _blobServiceClient.GetBlobContainerClient(containerName); + } + public BlobServiceClient GetBlobServiceClient() + { + return _blobServiceClient; + } + + private void ValidateConfiguration(StorageServiceConfiguration configuration) + { + Guard.Against.Null(configuration, nameof(configuration)); + + foreach (var key in ConfigurationKeys.RequiredKeys) + { + if (!configuration.Settings.ContainsKey(key)) + { + throw new ConfigurationException($"{nameof(AzureBlobClientFactory)} is missing configuration for {key}."); + } + } + } + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobHealthCheck.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobHealthCheck.cs new file mode 100644 index 0000000..e19ed86 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobHealthCheck.cs @@ -0,0 +1,53 @@ +/* + * Copyright 2022 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +namespace Monai.Deploy.Storage.AzureBlob +{ + internal class AzureBlobHealthCheck : IHealthCheck + { + private readonly IAzureBlobClientFactory _azureBlobClientFactory; + private readonly ILogger _logger; + + public AzureBlobHealthCheck(IAzureBlobClientFactory azureBlobClientFactory, ILogger logger) + { + _azureBlobClientFactory = azureBlobClientFactory ?? throw new ArgumentNullException(nameof(azureBlobClientFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = new()) + { + try + { + var client = _azureBlobClientFactory.GetBlobServiceClient(); + await client.GetBlobContainersAsync(cancellationToken: cancellationToken) + .AsPages(pageSizeHint: 1) + .GetAsyncEnumerator(cancellationToken) + .MoveNextAsync() + .ConfigureAwait(false); ; + + return HealthCheckResult.Healthy(); + } + catch (Exception exception) + { + _logger.HealthCheckError(exception); + return HealthCheckResult.Unhealthy(exception: exception); + } + } + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobStartup.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobStartup.cs new file mode 100644 index 0000000..450f3ac --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobStartup.cs @@ -0,0 +1,64 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Storage.Configuration; +using Azure.Storage.Blobs; +using Azure.Identity; + +namespace Monai.Deploy.Storage.AzureBlob +{ + public class AzureBlobStartup : IHostedService + { + private readonly Microsoft.Extensions.Options.IOptions _options; + private readonly ILogger _logger; + + public AzureBlobStartup(IOptions options, ILogger logger) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + if (_options.Value.Settings.ContainsKey(ConfigurationKeys.CreateBuckets)) + { + var buckets = _options.Value.Settings[ConfigurationKeys.CreateBuckets]; + + if (!string.IsNullOrWhiteSpace(buckets)) + { + var exceptions = new List(); + var bucketNames = buckets.Split(',', StringSplitOptions.RemoveEmptyEntries); + + var blobServiceClient = new BlobServiceClient(_options.Value.Settings[ConfigurationKeys.ConnectionString]); + + + foreach (var bucket in bucketNames) + { + try + { + await blobServiceClient.CreateBlobContainerAsync(bucket.Trim(), cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.ErrorCreatingContainer(bucket, ex); + exceptions.Add(ex); + } + } + + if (exceptions.Any()) + { + throw new AggregateException("Error creating buckets.", exceptions); + } + } + } + else + { + _logger.NoContainerCreated(); + } + } + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobStorageService.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobStorageService.cs new file mode 100644 index 0000000..1ef33c0 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/AzureBlobStorageService.cs @@ -0,0 +1,385 @@ +/* + * Copyright 2021-2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Amazon.SecurityToken.Model; +using Ardalis.GuardClauses; +using Azure.Storage.Blobs.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Storage.API; +using Monai.Deploy.Storage.Configuration; +using Azure.Storage.Blobs.Specialized; +using Azure.Storage.Sas; +using System.Text; + +namespace Monai.Deploy.Storage.AzureBlob +{ + public class AzureBlobStorageService : IStorageService + { + private readonly IAzureBlobClientFactory _azureBlobClientFactory; + private readonly ILogger _logger; + private readonly StorageServiceConfiguration _options; + + public string Name => "AzureBlob Storage Service"; + + public AzureBlobStorageService(IAzureBlobClientFactory azureBlobClientFactory, IOptions options, ILogger logger) + { + Guard.Against.Null(options); + _azureBlobClientFactory = azureBlobClientFactory ?? throw new ArgumentNullException(nameof(azureBlobClientFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var configuration = options.Value; + ValidateConfiguration(configuration); + + _options = configuration; + } + + private void ValidateConfiguration(StorageServiceConfiguration configuration) + { + Guard.Against.Null(configuration); + + foreach (var key in ConfigurationKeys.RequiredKeys) + { + if (!configuration.Settings.ContainsKey(key)) + { + throw new ConfigurationException($"{Name} is missing configuration for {key}."); + } + } + } + + #region ServiceAccount + + public async Task CopyObjectAsync(string sourcecontainer, string sourceObjectName, string destinationcontainer, string destinationObjectName, CancellationToken cancellationToken = default) + { + Guard.Against.NullOrWhiteSpace(sourceObjectName); + Guard.Against.NullOrWhiteSpace(destinationcontainer); + Guard.Against.NullOrWhiteSpace(destinationObjectName); + + var source = SanitiseBlobPath(sourcecontainer, sourceObjectName); + var destination = SanitiseBlobPath(destinationcontainer, destinationObjectName); + + try + { + var sourceClient = _azureBlobClientFactory.GetBlobClient(source.container, source.path); + var destClient = _azureBlobClientFactory.GetBlobBlockClient(destination.container, destination.path); + + var exsists = sourceClient.Exists(cancellationToken); + if (exsists.Value is false) + { + _logger.FileNotFoundError(sourcecontainer, sourceObjectName); + throw new StorageObjectNotFoundException($"Source file {sourceObjectName} does not exist in {sourcecontainer}"); + } + + var blobSasBuilder = new BlobSasBuilder() + { + BlobContainerName = source.container, + BlobName = source.path, + ExpiresOn = DateTime.UtcNow.AddHours(1) + }; + + blobSasBuilder.SetPermissions(BlobSasPermissions.Read); + var sasToken = sourceClient.GenerateSasUri(BlobSasPermissions.Read, DateTimeOffset.Now.AddHours(1)); + + + await destClient.StartCopyFromUriAsync(sourceClient.Uri /*, overwrite: false*/, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.BlobCopied(source.container, source.path, destination.container, destination.path); + } + catch (Exception ex) + { + _logger.StorageServiceError(ex); + throw new StorageServiceException(ex.Message); + } + } + + public async Task GetObjectAsync(string container, string objectName, CancellationToken cancellationToken = default) + { + Guard.Against.NullOrWhiteSpace(objectName); + + var source = SanitiseBlobPath(container, objectName); + try + { + var client = _azureBlobClientFactory.GetBlobClient(source.container, source.path); + + var stream = new MemoryStream(); + await client.DownloadToAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + stream.Seek(0, SeekOrigin.Begin); + _logger.BlobGetObject(source.container, source.path); + return stream; + } + catch (Exception ex) + { + _logger.StorageServiceError(ex); + throw new StorageServiceException(ex.Message); + } + } + + public async Task> ListObjectsAsync(string container, string? prefix = "", bool recursive = false, CancellationToken cancellationToken = default) + { + var maxSingle = 1000; + if (string.IsNullOrWhiteSpace(container)) { container = "$root"; } + + var source = SanitiseBlobPath(container, prefix ?? ""); + var folder = Path.GetDirectoryName(source.path) ?? ""; + + try + { + var client = _azureBlobClientFactory.GetBlobContainerClient(source.container); + var containerExists = await client.ExistsAsync().ConfigureAwait(false); + if (containerExists.Value is false) + { + _logger.ContainerDoesNotExistCreated(source.container); + return new List(); + } + var resultSegment = client.GetBlobsAsync(prefix: source.path, cancellationToken: cancellationToken).AsPages(default, maxSingle); + var files = new List(); + + await foreach (var blobPage in resultSegment) + { + foreach (var blobItem in blobPage.Values) + { + var file = new VirtualFileInfo(Path.GetFileName(blobItem.Name), + blobItem.Name ?? "", + string.Empty, + (ulong)(blobItem.Properties.ContentLength ?? 0 + )); + + if (recursive) + { + files.Add(file); + } + else if (file.FilePath == folder) + { + files.Add(file); + } + + }; + } + _logger.BlobListObjects(container, prefix); + return files; + } + catch (Exception ex) + { + _logger.StorageServiceError(ex); + throw new StorageServiceException(ex.Message); + } + } + + public async Task> VerifyObjectsExistAsync(string container, IReadOnlyList artifactList, CancellationToken cancellationToken = default) + { + Guard.Against.Null(artifactList); + + var existingObjectsDict = new Dictionary(); + var exceptions = new List(); + + foreach (var artifact in artifactList) + { + try + { + var source = SanitiseBlobPath(container, artifact); + var blobClient = _azureBlobClientFactory.GetBlobClient(source.container, source.path); + + var exists = await blobClient.ExistsAsync(); + if (exists.Value is false) + { + _logger.FileNotFoundError(container, $"{artifact}"); + + existingObjectsDict.Add(artifact, false); + continue; + } + existingObjectsDict.Add(artifact, true); + } + catch (Exception e) + { + _logger.VerifyObjectError(container, e); + existingObjectsDict.Add(artifact, false); + exceptions.Add(e); + } + } + + if (exceptions.Any()) + { + throw new VerifyObjectsException(exceptions, existingObjectsDict); + } + return existingObjectsDict; + } + + public async Task VerifyObjectExistsAsync(string container, string artifactName, CancellationToken cancellationToken = default) + { + Guard.Against.NullOrWhiteSpace(artifactName); + + try + { + var source = SanitiseBlobPath(container, artifactName); + var blobClient = _azureBlobClientFactory.GetBlobClient(source.container, source.path); + var exists = await blobClient.ExistsAsync(); + + if (exists.Value is true) + { + return true; + } + + _logger.FileNotFoundError(container, $"{artifactName}"); + + return false; + } + catch (Exception ex) + { + _logger.VerifyObjectError(container, ex); + throw new VerifyObjectsException(ex.Message, ex); + } + } + + public async Task PutObjectAsync(string container, string objectName, Stream data, long size, string contentType, Dictionary? metadata, CancellationToken cancellationToken = default) + { + Guard.Against.NullOrWhiteSpace(objectName); + Guard.Against.Null(data); + Guard.Against.NullOrWhiteSpace(contentType); + + var source = SanitiseBlobPath(container, objectName); + + try + { + var client = _azureBlobClientFactory.GetBlobClient(source.container, source.path); + var headers = new BlobHttpHeaders { ContentType = contentType }; + await client.UploadAsync(data, overwrite: false, cancellationToken: cancellationToken).ConfigureAwait(false); + await client.SetHttpHeadersAsync(headers, cancellationToken: cancellationToken).ConfigureAwait(false); + await client.SetMetadataAsync(metadata, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.BlobPutObject(source.container, source.path); + } + catch (Exception ex) + { + _logger.StorageServiceError(ex); + throw new StorageServiceException(ex.Message); + } + } + + public async Task RemoveObjectAsync(string container, string objectName, CancellationToken cancellationToken = default) + { + Guard.Against.NullOrWhiteSpace(objectName); + var source = SanitiseBlobPath(container, objectName); + + try + { + var client = _azureBlobClientFactory.GetBlobClient(source.container, source.path); + await client.DeleteAsync().ConfigureAwait(false); + _logger.BlobRemoveObject(source.container, source.path); + } + catch (Exception ex) + { + _logger.StorageServiceError(ex); + //throw new StorageServiceException(ex.Message); + } + } + + public async Task RemoveObjectsAsync(string container, IEnumerable objectNames, CancellationToken cancellationToken = default) + { + Guard.Against.NullOrEmpty(objectNames); + + var sanitisedNames = objectNames.Select(name => SanitiseBlobPath(container, name).path); + + try + { + var containerClient = _azureBlobClientFactory.GetBlobContainerClient(container); + var batchClient = new BlobBatchClient(containerClient); + + await batchClient.DeleteBlobsAsync(sanitisedNames.Select(s => new Uri($"{containerClient.Uri}/{s}"))).ConfigureAwait(false); + _logger.BlobRemoveObjects(container); + } + catch (Exception ex) + { + _logger.StorageServiceError(ex); + //throw new StorageServiceException(ex.Message); + } + } + + public async Task CreateFolderAsync(string container, string folderPath, CancellationToken cancellationToken = default) + { + var containerPath = SanitiseBlobPath(container, folderPath); + try + { + var containerClient = _azureBlobClientFactory.GetBlobContainerClient(containerPath.container); + await containerClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + var path = $"{containerPath.path}/stubFile.txt".Replace("//", "/"); + var blobClient = _azureBlobClientFactory.GetBlobClient(containerPath.container, path); + + var data = Encoding.UTF8.GetBytes("stub file"); + var length = data.Length; + var stream = new MemoryStream(data); + + await blobClient.UploadAsync(stream, true, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.ContainerCreated($"{containerPath.container}/{containerPath.path}"); + } + catch (Exception ex) + { + _logger.StorageServiceError(ex); + throw new StorageServiceException(ex.Message); + } + } + + #endregion ServiceAccount + + #region TemporaryCredentials + + public async Task CopyObjectWithCredentialsAsync(string sourcecontainer, string sourceObjectName, string destinationcontainer, string destinationObjectName, Credentials credentials, CancellationToken cancellationToken = default) + { + await CopyObjectAsync(sourcecontainer, sourceObjectName, destinationcontainer, destinationObjectName, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + public async Task GetObjectWithCredentialsAsync(string container, string objectName, Credentials credentials, CancellationToken cancellationToken = default) + { + return await GetObjectAsync(container, objectName, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + public async Task> ListObjectsWithCredentialsAsync(string container, Credentials credentials, string? prefix = "", bool recursive = false, CancellationToken cancellationToken = default) + { + return await ListObjectsAsync(container, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + public async Task PutObjectWithCredentialsAsync(string container, string objectName, Stream data, long size, string contentType, Dictionary metadata, Credentials credentials, CancellationToken cancellationToken = default) + { + await PutObjectAsync(container, objectName, data, size, contentType, metadata, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + public async Task RemoveObjectWithCredentialsAsync(string container, string objectName, Credentials credentials, CancellationToken cancellationToken = default) + { + await RemoveObjectAsync(container, objectName, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + public async Task RemoveObjectsWithCredentialsAsync(string container, IEnumerable objectNames, Credentials credentials, CancellationToken cancellationToken = default) + { + await RemoveObjectsAsync(container, objectNames, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + public async Task CreateFolderWithCredentialsAsync(string container, string folderPath, Credentials credentials, CancellationToken cancellationToken = default) + { + await CreateFolderAsync(container, folderPath, cancellationToken).ConfigureAwait(false); + } + + #endregion TemporaryCredentials + + public Task CreateTemporaryCredentialsAsync(string container, string folderName, int durationSeconds = 3600, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + + private (string container, string path) SanitiseBlobPath(string container, string path) + { + if (string.IsNullOrWhiteSpace(container)) { return ("", path); } + var whole = $"{container}/{path}".Replace("//", "/").Replace("//", "/"); + var indexOf = whole.IndexOf("/"); + var containerBit = container.Substring(0, indexOf); + var pathBit = whole.Substring(indexOf + 1); + return (containerBit, pathBit); + } + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/ConfigurationKeys.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/ConfigurationKeys.cs new file mode 100644 index 0000000..5bb0b16 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/ConfigurationKeys.cs @@ -0,0 +1,30 @@ +/* + * Copyright 2021-2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Monai.Deploy.Storage.AzureBlob +{ + internal static class ConfigurationKeys + { + public static readonly string StorageServiceName = "azureblob"; + + public static readonly string ConnectionString = "azureBlobConnectionString"; + public static readonly string CreateBuckets = "createBuckets"; + public static readonly string ApiCallTimeout = "timeout"; + + public static readonly string[] RequiredKeys = new[] { ConnectionString }; + + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/HealthCheckBuilder.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/HealthCheckBuilder.cs new file mode 100644 index 0000000..ca3c9ee --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/HealthCheckBuilder.cs @@ -0,0 +1,40 @@ +/* + * Copyright 2022 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +//using Monai.Deploy.Storage.MinIO; + +namespace Monai.Deploy.Storage.AzureBlob +{ + public class HealthCheckBuilder : HealthCheckRegistrationBase + { + public override IHealthChecksBuilder ConfigureAdminHealthCheck(IHealthChecksBuilder builder, HealthStatus? failureStatus = null, IEnumerable? tags = null, TimeSpan? timeout = null) => throw new NotImplementedException(); + public override IHealthChecksBuilder ConfigureHealthCheck(IHealthChecksBuilder builder, HealthStatus? failureStatus = null, IEnumerable? tags = null, TimeSpan? timeout = null) => + builder.Add(new HealthCheckRegistration( + ConfigurationKeys.StorageServiceName, + serviceProvider => + { + var logger = serviceProvider.GetRequiredService>(); + var azureBlobClientFactory = serviceProvider.GetRequiredService(); + return new AzureBlobHealthCheck(azureBlobClientFactory, logger); + }, + failureStatus, + tags, + timeout)); + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/IAzureBlobClientFactory.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/IAzureBlobClientFactory.cs new file mode 100644 index 0000000..963fa86 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/IAzureBlobClientFactory.cs @@ -0,0 +1,15 @@ +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Specialized; + +namespace Monai.Deploy.Storage.AzureBlob +{ + public interface IAzureBlobClientFactory + { + BlockBlobClient GetBlobBlockClient(BlobContainerClient containerClient, string blob); + BlockBlobClient GetBlobBlockClient(string containerName, string blob); + BlobClient GetBlobClient(BlobContainerClient containerClient, string blob); + BlobClient GetBlobClient(string containerName, string blob); + BlobContainerClient GetBlobContainerClient(string containerName); + BlobServiceClient GetBlobServiceClient(); + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/InternalsVisibleTo.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/InternalsVisibleTo.cs new file mode 100644 index 0000000..168d640 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/InternalsVisibleTo.cs @@ -0,0 +1,20 @@ +/* + * Copyright 2022 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Monai.Deploy.Storage.AzureBlob.Tests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/LoggerMethods.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/LoggerMethods.cs new file mode 100644 index 0000000..b0ef8b8 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/LoggerMethods.cs @@ -0,0 +1,72 @@ +/* + * Copyright 2023 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Microsoft.Extensions.Logging; + +namespace Monai.Deploy.Storage.AzureBlob +{ + public static partial class LoggerMethods + { + [LoggerMessage(EventId = 20000, Level = LogLevel.Error, Message = "Error listing objects in container '{containerName}' with error: {error}")] + public static partial void ListObjectError(this ILogger logger, string containerName, string error); + + [LoggerMessage(EventId = 20001, Level = LogLevel.Error, Message = "File '{path}' could not be found in '{containerName}'.")] + public static partial void FileNotFoundError(this ILogger logger, string containerName, string path); + + [LoggerMessage(EventId = 20002, Level = LogLevel.Error, Message = "Error verifying objects in container '{containerName}'.")] + public static partial void VerifyObjectError(this ILogger logger, string containerName, Exception ex); + + [LoggerMessage(EventId = 20003, Level = LogLevel.Error, Message = "Health check failure.")] + public static partial void HealthCheckError(this ILogger logger, Exception ex); + + [LoggerMessage(EventId = 20004, Level = LogLevel.Debug, Message = "Temporary credential policy={policy}.")] + public static partial void TemporaryCredentialPolicy(this ILogger logger, string policy); + + [LoggerMessage(EventId = 20005, Level = LogLevel.Information, Message = "`createcontainers` not configured; no containers created.")] + public static partial void NoContainerCreated(this ILogger logger); + + [LoggerMessage(EventId = 20006, Level = LogLevel.Critical, Message = "Error creating container {container}.")] + public static partial void ErrorCreatingContainer(this ILogger logger, string container, Exception ex); + + [LoggerMessage(EventId = 20007, Level = LogLevel.Information, Message = "container {container} created")] + public static partial void ContainerCreated(this ILogger logger, string container); + + [LoggerMessage(EventId = 20009, Level = LogLevel.Error, Message = "Storage service error.")] + public static partial void StorageServiceError(this ILogger logger, Exception ex); + + [LoggerMessage(EventId = 20010, Level = LogLevel.Debug, Message = "Copied from {sourceContainer}/{sourcePath} to {destinationContainer}/{destinationPath}")] + public static partial void BlobCopied(this ILogger logger, string sourceContainer, string sourcePath, string destinationContainer, string destinationPath); + + [LoggerMessage(EventId = 20011, Level = LogLevel.Debug, Message = "Returning stream from {sourceContainer}/{sourcePath}")] + public static partial void BlobGetObject(this ILogger logger, string sourceContainer, string sourcePath); + + [LoggerMessage(EventId = 20012, Level = LogLevel.Trace, Message = "Returning file list from {sourceContainer} with prefix {prefix}")] + public static partial void BlobListObjects(this ILogger logger, string sourceContainer, string? prefix); + + [LoggerMessage(EventId = 20013, Level = LogLevel.Debug, Message = "Uploaded file {sourceContainer}/{path}")] + public static partial void BlobPutObject(this ILogger logger, string sourceContainer, string path); + + [LoggerMessage(EventId = 20014, Level = LogLevel.Debug, Message = "Remove file {sourceContainer}/{path}")] + public static partial void BlobRemoveObject(this ILogger logger, string sourceContainer, string path); + + [LoggerMessage(EventId = 20015, Level = LogLevel.Debug, Message = "Remove a list of files from {sourceContainer}")] + public static partial void BlobRemoveObjects(this ILogger logger, string sourceContainer); + + [LoggerMessage(EventId = 20016, Level = LogLevel.Error, Message = "container {container} does not exist")] + public static partial void ContainerDoesNotExistCreated(this ILogger logger, string container); + + } +} diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/Monai.Deploy.Storage.AzureBlob.csproj b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/Monai.Deploy.Storage.AzureBlob.csproj new file mode 100644 index 0000000..f9b7b99 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/Monai.Deploy.Storage.AzureBlob.csproj @@ -0,0 +1,21 @@ + + + + net6.0 + enable + enable + + + + + + + + + + + + + + + diff --git a/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/ServiceRegistration.cs b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/ServiceRegistration.cs new file mode 100644 index 0000000..166d760 --- /dev/null +++ b/src/Plugins/AzureBlob/Monai.Deploy.Storage.AzureBlob/ServiceRegistration.cs @@ -0,0 +1,31 @@ +/* + * Copyright 2022 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Microsoft.Extensions.DependencyInjection; + +namespace Monai.Deploy.Storage.AzureBlob +{ + public class ServiceRegistration : ServiceRegistrationBase + { + public override IServiceCollection Configure(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(p => p.GetRequiredService()); + return services; + } + } +} diff --git a/src/Plugins/MinIO/LoggerMethods.cs b/src/Plugins/MinIO/LoggerMethods.cs index 3d92048..52e96ff 100644 --- a/src/Plugins/MinIO/LoggerMethods.cs +++ b/src/Plugins/MinIO/LoggerMethods.cs @@ -50,5 +50,6 @@ public static partial class LoggerMethods [LoggerMessage(EventId = 20009, Level = LogLevel.Error, Message = "Storage service error.")] public static partial void StorageServiceError(this ILogger logger, Exception ex); + } }