diff --git a/global.json b/global.json index 32da22419b..ce2e41208c 100644 --- a/global.json +++ b/global.json @@ -4,4 +4,4 @@ "rollForward": "latestFeature", "allowPrerelease": false } -} \ No newline at end of file +} diff --git a/src/NuGet.Services.Storage/AggregateStorage.cs b/src/NuGet.Services.Storage/AggregateStorage.cs index 4fc61ab402..3bbf0f06dd 100644 --- a/src/NuGet.Services.Storage/AggregateStorage.cs +++ b/src/NuGet.Services.Storage/AggregateStorage.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -111,9 +111,12 @@ public override async Task> ListAsync(bool getMetad return await _primaryStorage.ListAsync(getMetadata, cancellationToken); } + public override async Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) => + await _primaryStorage.ListTopLevelAsync(getMetadata, cancellationToken); + public override Task SetMetadataAsync(Uri resourceUri, IDictionary metadata) { throw new NotImplementedException(); } } -} \ No newline at end of file +} diff --git a/src/NuGet.Services.Storage/AzureStorage.cs b/src/NuGet.Services.Storage/AzureStorage.cs index 8e54874625..1c30798c24 100644 --- a/src/NuGet.Services.Storage/AzureStorage.cs +++ b/src/NuGet.Services.Storage/AzureStorage.cs @@ -189,7 +189,29 @@ public override async Task> ListAsync(bool getMetad await foreach (BlobHierarchyItem blob in _directory.GetBlobsByHierarchyAsync(traits: blobTraits, prefix: _path)) { - blobList.Add(await GetStorageListItemAsync(_directory.GetBlockBlobClient(blob.Blob.Name))); + blobList.Add(await GetStorageListItemAsync(_directory.GetBlockBlobClient(blob.Blob.Name))); + } + + return blobList; + } + + public override async Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) + { + var prefix = _path.Trim('/') + '/'; + var blobTraits = new BlobTraits(); + if (getMetadata) + { + blobTraits |= BlobTraits.Metadata; + } + + var blobList = new List(); + + await foreach (BlobHierarchyItem blob in _directory.GetBlobsByHierarchyAsync(traits: blobTraits, prefix: prefix, delimiter: "/")) + { + if (!blob.IsPrefix) + { + blobList.Add(await GetStorageListItemAsync(_directory.GetBlockBlobClient(blob.Blob.Name))); + } } return blobList; diff --git a/src/NuGet.Services.Storage/FileStorage.cs b/src/NuGet.Services.Storage/FileStorage.cs index 32732ab85f..ea724df432 100644 --- a/src/NuGet.Services.Storage/FileStorage.cs +++ b/src/NuGet.Services.Storage/FileStorage.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -57,6 +57,11 @@ public override Task> ListAsync(bool getMetadata, C return Task.FromResult>(List(getMetadata)); } + public override Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + public string Path { get; diff --git a/src/NuGet.Services.Storage/IStorage.cs b/src/NuGet.Services.Storage/IStorage.cs index ccd11b146e..6e00bceb2e 100644 --- a/src/NuGet.Services.Storage/IStorage.cs +++ b/src/NuGet.Services.Storage/IStorage.cs @@ -1,7 +1,8 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Collections; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -19,7 +20,13 @@ public interface IStorage Uri BaseAddress { get; } Uri ResolveUri(string relativeUri); IEnumerable List(bool getMetadata); + + //Lists all children of the storage(including the ones contained in subdirectories). Task> ListAsync(bool getMetadata, CancellationToken cancellationToken); + + //Lists immediate children of the storage assuming directory-like structure + Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken); + Task CopyAsync( Uri sourceUri, IStorage destinationStorage, diff --git a/src/NuGet.Services.Storage/Storage.cs b/src/NuGet.Services.Storage/Storage.cs index 2f428d0501..ae66aeb4fe 100644 --- a/src/NuGet.Services.Storage/Storage.cs +++ b/src/NuGet.Services.Storage/Storage.cs @@ -156,6 +156,7 @@ public async Task LoadString(Uri resourceUri, CancellationToken cancella public abstract Task ExistsAsync(string fileName, CancellationToken cancellationToken); public abstract IEnumerable List(bool getMetadata); public abstract Task> ListAsync(bool getMetadata, CancellationToken cancellationToken); + public abstract Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken); public bool Verbose { @@ -209,6 +210,12 @@ protected string GetName(Uri uri) { name = name.Substring(0, name.IndexOf("#")); } + + if (name.Contains("?")) + { + name = name.Substring(0, name.IndexOf("?")); + } + return name; } diff --git a/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs b/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs index 8441fa4d2e..13a720ad4a 100644 --- a/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs +++ b/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -10,6 +10,7 @@ using System.Net; using System.Threading; using System.Threading.Tasks; +using Azure.Core; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Newtonsoft.Json; @@ -120,7 +121,7 @@ private async Task ProcessBlobs(List jsonBlobs, CancellationTok foreach (var sourceBlob in jsonBlobs) { var blobName = GetBlobName(sourceBlob); - var workBlobUri = _workStorage.ResolveUri(blobName); + var workBlobUri = _workStorage.ResolveUri(_configuration.WorkPath + blobName); var sourceBlobStats = new BlobStatistics(); var individualReports = await ProcessSourceBlobAsync(sourceBlob, sourceBlobStats, totals); using (_logger.BeginScope("Processing {BlobName}", blobName)) @@ -154,7 +155,7 @@ private async Task ProcessBlobs(List jsonBlobs, CancellationTok } } } - var jobSucceededUrl = _workStorage.ResolveUri(JobSucceededFilename); + var jobSucceededUrl = _workStorage.ResolveUri(_configuration.WorkPath + JobSucceededFilename); var jobSucceededContent = new StringStorageContent("", TextContentType); await _workStorage.Save(jobSucceededUrl, jobSucceededContent, overwrite: true, cancellationToken: cancellationToken); _telemetryService.ReportTotals(totals.SourceFilesProcessed, totals.TotalLinesProcessed, totals.TotalFilesCreated, totals.TotalLinesFailed); @@ -202,26 +203,26 @@ private async Task CopySourceBlobsAsync(List jsonBlobs, Cancell foreach (var sourceBlob in jsonBlobs) { var blobName = GetBlobName(sourceBlob); - var targetUrl = _workStorage.ResolveUri(blobName); + var targetUrl = _workStorage.ResolveUri(_configuration.WorkPath + blobName); _logger.LogInformation("{SourceBlobUri} ({BlobName})", sourceBlob.Uri.AbsoluteUri, blobName); _logger.LogInformation("{WorkBlobUrl}", targetUrl); await _sourceStorage.CopyAsync(sourceBlob.Uri, _workStorage, targetUrl, destinationProperties: null, cancellationToken); } var copySucceededContent = new StringStorageContent("", TextContentType); - var copySucceededUrl = _workStorage.ResolveUri(CopySucceededFilename); + var copySucceededUrl = _workStorage.ResolveUri(_configuration.WorkPath + CopySucceededFilename); await _workStorage.Save(copySucceededUrl, copySucceededContent, overwrite: true, cancellationToken: cancellationToken); } private async Task> EnumerateSourceBlobsAsync() { - var blobs = await _sourceStorage.ListAsync(getMetadata: true, cancellationToken: CancellationToken.None); + var blobs = await _sourceStorage.ListTopLevelAsync(getMetadata: true, cancellationToken: CancellationToken.None); return blobs.ToList(); } private async Task> EnumerateWorkStorageBlobsAsync() { - var blobs = await _workStorage.ListAsync(getMetadata: true, cancellationToken: CancellationToken.None); + var blobs = await _workStorage.ListTopLevelAsync(getMetadata: true, cancellationToken: CancellationToken.None); return blobs.ToList(); } @@ -245,7 +246,7 @@ private async Task> ProcessSourceBlobAsync( var sw = Stopwatch.StartNew(); var numLines = 0; var individualReports = new ConcurrentBag(); - var workStorageUrl = _workStorage.ResolveUri(GetBlobName(sourceBlob)); + var workStorageUrl = _workStorage.ResolveUri(_configuration.WorkPath + GetBlobName(sourceBlob)); var storageContent = await _workStorage.Load(workStorageUrl, CancellationToken.None); using (var sourceStream = storageContent.GetContentStream()) using (var streamReader = new StreamReader(sourceStream)) @@ -311,7 +312,7 @@ private static bool BlobMetadataExists(StorageListItem sourceBlob, TotalStats to private static string GetBlobName(StorageListItem blob) { - var path = blob.Uri.AbsoluteUri; + var path = blob.Uri.GetComponents(UriComponents.Path, UriFormat.UriEscaped); var lastSlash = path.LastIndexOf('/'); if (lastSlash < 0) { @@ -351,7 +352,7 @@ private async Task WriteReports( continue; } var outFilename = $"recentpopularitydetail_{data.PackageId.ToLowerInvariant()}.json"; - var destinationUri = _destinationStorage.ResolveUri(outFilename); + var destinationUri = _destinationStorage.ResolveUri(_configuration.DestinationPath + outFilename); var storageContent = new StringStorageContent(details.Data, JsonContentType); await _destinationStorage.Save(destinationUri, storageContent, overwrite: true, cancellationToken: cancellationToken); diff --git a/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs b/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs index 12791043e2..f45a184924 100644 --- a/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs +++ b/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs @@ -165,6 +165,11 @@ public override Task> ListAsync(bool getMetadata, C return _storage.ListAsync(getMetadata, cancellationToken); } + public override Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) + { + return _storage.ListTopLevelAsync(getMetadata, cancellationToken); + } + public override Task SetMetadataAsync(Uri resourceUri, IDictionary metadata) { return _storage.SetMetadataAsync(resourceUri, metadata); diff --git a/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs b/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs index 67d77e7f38..8acd723d96 100644 --- a/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs +++ b/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -33,13 +33,13 @@ public class DetailedReportPostProcessorFacts public async Task DoesntStartIfNoSuccessFile() { _sourceStorageMock - .Setup(ss => ss.ListAsync(It.IsAny(), It.IsAny())) + .Setup(ss => ss.ListTopLevelAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(new List()); await _target.CopyReportsAsync(); _sourceStorageMock - .Verify(ss => ss.ListAsync(It.IsAny(), It.IsAny()), Times.Once); + .Verify(ss => ss.ListTopLevelAsync(It.IsAny(), It.IsAny()), Times.Once); _sourceStorageMock.VerifyNoOtherCalls(); _workStorageMock.VerifyNoOtherCalls(); _destinationStorageMock.VerifyNoOtherCalls(); @@ -186,7 +186,7 @@ public async Task SkipsProcessedFiles() { "FilesCreated", "123" } }; _workStorageMock - .Setup(ss => ss.ListAsync(It.IsAny(), It.IsAny())) + .Setup(ss => ss.ListTopLevelAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(() => new List(_workFiles.Select(f => Blob( _workStorageMock, f, @@ -286,7 +286,7 @@ private static void SetupStorageMock(Mock mock, string baseUrl, Func s.ListAsync(It.IsAny(), It.IsAny())) + .Setup(s => s.ListTopLevelAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(() => new List(files().Select(f => Blob(mock, f)))); mock .Setup(s => s.ResolveUri(It.IsAny()))