diff --git a/KernelMemory.sln.DotSettings b/KernelMemory.sln.DotSettings index d9a10ac0b..cc1462402 100644 --- a/KernelMemory.sln.DotSettings +++ b/KernelMemory.sln.DotSettings @@ -271,6 +271,7 @@ public void It$SOMENAME$() True DO_NOT_SHOW True + True True True True diff --git a/clients/dotnet/WebClient/MemoryWebClient.cs b/clients/dotnet/WebClient/MemoryWebClient.cs index 730968374..2e60cb343 100644 --- a/clients/dotnet/WebClient/MemoryWebClient.cs +++ b/clients/dotnet/WebClient/MemoryWebClient.cs @@ -233,6 +233,13 @@ public async Task IsDocumentReadyAsync( return status; } +#if KernelMemoryDev + public Task ExportFileAsync(string documentId, string fileName, string? index = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } +#endif + /// public async Task SearchAsync( string query, diff --git a/clients/dotnet/WebClient/WebClient.csproj b/clients/dotnet/WebClient/WebClient.csproj index b69d40942..2f673a08d 100644 --- a/clients/dotnet/WebClient/WebClient.csproj +++ b/clients/dotnet/WebClient/WebClient.csproj @@ -5,6 +5,7 @@ Microsoft.KernelMemory.WebClient Microsoft.KernelMemory $(NoWarn);CS1591;NU5104; + $(DefineConstants);KernelMemoryDev diff --git a/examples/006-curl-calling-webservice/README.md b/examples/006-curl-calling-webservice/README.md index ac82f947a..48a5762ff 100644 --- a/examples/006-curl-calling-webservice/README.md +++ b/examples/006-curl-calling-webservice/README.md @@ -18,7 +18,7 @@ Content of [upload-example.sh](upload-example.sh): ```bash ../../tools/upload-file.sh -f test.pdf \ -s http://127.0.0.1:9001 \ - -u curlUser \ + -p curlUser \ -t "type:test" \ -i curlExample01 ``` @@ -33,7 +33,7 @@ Content of [ask-example.sh](ask-example.sh): ```bash ../../tools/ask.sh -s http://127.0.0.1:9001 \ - -u curlUser \ + -p curlUser \ -q "tell me about Semantic Kernel" \ -f '"type":["test"]' ``` \ No newline at end of file diff --git a/extensions/AzureBlobs/AzureBlobs.csproj b/extensions/AzureBlobs/AzureBlobs.csproj index a3786f6cf..35ca67431 100644 --- a/extensions/AzureBlobs/AzureBlobs.csproj +++ b/extensions/AzureBlobs/AzureBlobs.csproj @@ -6,6 +6,7 @@ Microsoft.KernelMemory.ContentStorage.AzureBlobs Microsoft.KernelMemory.ContentStorage.AzureBlobs $(NoWarn);CA1724;CS1591; + $(DefineConstants);KernelMemoryDev diff --git a/extensions/AzureBlobs/AzureBlobsStorage.cs b/extensions/AzureBlobs/AzureBlobsStorage.cs index 0a81579a9..e1ee29b62 100644 --- a/extensions/AzureBlobs/AzureBlobsStorage.cs +++ b/extensions/AzureBlobs/AzureBlobsStorage.cs @@ -189,6 +189,18 @@ public Task WriteFileAsync( return this.InternalWriteAsync(directoryName, fileName, streamContent, cancellationToken); } +#if KernelMemoryDev + /// + public Task ReadFileAsync( + string index, + string documentId, + string fileName, + bool logErrIfNotFound = true, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } +#else /// public async Task ReadFileAsync( string index, @@ -220,6 +232,7 @@ public async Task ReadFileAsync( throw new ContentStorageFileNotFoundException("File not found", e); } } +#endif #region private diff --git a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlas.csproj b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlas.csproj index b5f9a62fd..78f42b5a8 100644 --- a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlas.csproj +++ b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlas.csproj @@ -6,6 +6,7 @@ Microsoft.KernelMemory.MongoDbAtlas Microsoft.KernelMemory.MongoDbAtlas $(NoWarn);CA1724;CS1591;CA1308; + $(DefineConstants);KernelMemoryDev diff --git a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs index f245366f7..f9b4a7fab 100644 --- a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs +++ b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs @@ -124,6 +124,19 @@ public Task CreateDocumentDirectoryAsync(string index, string documentId, return Task.CompletedTask; } +#if KernelMemoryDev + /// + public Task ReadFileAsync( + string index, + string documentId, + string fileName, + bool logErrIfNotFound = true, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } +#else + /// public async Task ReadFileAsync(string index, string documentId, string fileName, bool logErrIfNotFound = true, CancellationToken cancellationToken = new CancellationToken()) { @@ -189,6 +202,7 @@ public async Task ReadFileAsync(string index, string documentId, str return new BinaryData(memoryStream.ToArray()); } } +#endif private async Task SaveDocumentAsync(string index, string id, BsonDocument doc, CancellationToken cancellationToken) { diff --git a/extensions/TikToken/TikToken/TikToken.csproj b/extensions/TikToken/TikToken/TikToken.csproj index 735ccdd47..7bfc04884 100644 --- a/extensions/TikToken/TikToken/TikToken.csproj +++ b/extensions/TikToken/TikToken/TikToken.csproj @@ -5,7 +5,7 @@ LatestMajor Microsoft.KernelMemory.AI.TikToken Microsoft.KernelMemory.AI.TikToken - $(NoWarn); + $(NoWarn);NU5104; diff --git a/nuget-package.props b/nuget-package.props index 4c182da8d..3d7ade528 100644 --- a/nuget-package.props +++ b/nuget-package.props @@ -1,7 +1,7 @@  - 0.39.0 + 0.40.0 false diff --git a/service/Abstractions/Constants.cs b/service/Abstractions/Constants.cs index 5dfb4ad2e..b9a13a5f3 100644 --- a/service/Abstractions/Constants.cs +++ b/service/Abstractions/Constants.cs @@ -13,6 +13,9 @@ public static class Constants // Form field containing the Document ID public const string WebServiceDocumentIdField = "documentId"; + // Form field containing the Filename + public const string WebServiceFilenameField = "filename"; + // Form field containing the list of tags public const string WebServiceTagsField = "tags"; @@ -50,6 +53,7 @@ public static class Constants // Endpoints public const string HttpAskEndpoint = "/ask"; public const string HttpSearchEndpoint = "/search"; + public const string HttpDownloadEndpoint = "/download"; public const string HttpUploadEndpoint = "/upload"; public const string HttpUploadStatusEndpoint = "/upload-status"; public const string HttpDocumentsEndpoint = "/documents"; diff --git a/service/Abstractions/ContentStorage/IContentStorage.cs b/service/Abstractions/ContentStorage/IContentStorage.cs index 7e3f9e6d1..ab93b0d0d 100644 --- a/service/Abstractions/ContentStorage/IContentStorage.cs +++ b/service/Abstractions/ContentStorage/IContentStorage.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -84,7 +83,7 @@ Task WriteFileAsync( /// Whether to log an error if the file does not exist. An exception will be raised anyway. /// Async task cancellation token /// File content - Task ReadFileAsync( + Task ReadFileAsync( string index, string documentId, string fileName, diff --git a/service/Abstractions/IKernelMemory.cs b/service/Abstractions/IKernelMemory.cs index a2b0c8d7e..bf7bed55d 100644 --- a/service/Abstractions/IKernelMemory.cs +++ b/service/Abstractions/IKernelMemory.cs @@ -160,6 +160,20 @@ public Task IsDocumentReadyAsync( string? index = null, CancellationToken cancellationToken = default); + /// + /// Export a file from content storage + /// + /// ID of the document containing the file + /// File name + /// Index containing the document + /// Async task cancellation token + /// File content + public Task ExportFileAsync( + string documentId, + string fileName, + string? index = null, + CancellationToken cancellationToken = default); + /// /// Search the given index for a list of relevant documents for the given query. /// diff --git a/service/Abstractions/Models/StreamableFileContent.cs b/service/Abstractions/Models/StreamableFileContent.cs new file mode 100644 index 000000000..60e5ae1ef --- /dev/null +++ b/service/Abstractions/Models/StreamableFileContent.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Microsoft.KernelMemory; + +public sealed class StreamableFileContent : IDisposable +{ + private Stream? _stream; + + public string FileName { get; } = string.Empty; + public long FileSize { get; } = 0; + public string FileType { get; } = string.Empty; + public DateTimeOffset LastWrite { get; } = default; + public Func> StreamAsync { get; } + + public StreamableFileContent() + { + this.StreamAsync = () => Task.FromResult(new MemoryStream()); + } + + public StreamableFileContent( + string fileName, + long fileSize, + string fileType = "application/octet-stream", + DateTimeOffset lastWriteTimeUtc = default, + Func>? asyncStreamDelegate = null) + { + ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(fileType, nameof(fileType), "File content type is empty"); + ArgumentNullExceptionEx.ThrowIfNull(lastWriteTimeUtc, nameof(lastWriteTimeUtc), "File last write time is NULL"); + ArgumentNullExceptionEx.ThrowIfNull(asyncStreamDelegate, nameof(asyncStreamDelegate), "asyncStreamDelegate is NULL"); + + this.FileName = fileName; + this.FileSize = fileSize; + this.FileType = fileType; + this.LastWrite = lastWriteTimeUtc; + this.StreamAsync = async () => + { + this._stream = await asyncStreamDelegate().ConfigureAwait(false); + return this._stream; + }; + } + + public void Dispose() + { + if (this._stream == null) { return; } + + this._stream.Close(); + this._stream.Dispose(); + } +} diff --git a/service/Abstractions/Pipeline/IPipelineOrchestrator.cs b/service/Abstractions/Pipeline/IPipelineOrchestrator.cs index 12cb85739..52ff6d8a3 100644 --- a/service/Abstractions/Pipeline/IPipelineOrchestrator.cs +++ b/service/Abstractions/Pipeline/IPipelineOrchestrator.cs @@ -91,6 +91,15 @@ public interface IPipelineOrchestrator /// Task StopAllPipelinesAsync(); + /// + /// Fetch a file from content storage, streaming its content and details + /// + /// Pipeline containing the file + /// Name of the file to fetch + /// Async task cancellation token + /// File data + Task ReadFileAsStreamAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default); + /// /// Fetch a file from content storage /// diff --git a/service/Core/ContentStorage/DevTools/SimpleFileStorage.cs b/service/Core/ContentStorage/DevTools/SimpleFileStorage.cs index 89c9a3e43..69484f051 100644 --- a/service/Core/ContentStorage/DevTools/SimpleFileStorage.cs +++ b/service/Core/ContentStorage/DevTools/SimpleFileStorage.cs @@ -91,6 +91,18 @@ public async Task WriteFileAsync( await this._fileSystem.WriteFileAsync(volume: index, relPath: documentId, fileName: fileName, streamContent: streamContent, cancellationToken).ConfigureAwait(false); } +#if KernelMemoryDev + /// + public Task ReadFileAsync( + string index, + string documentId, + string fileName, + bool logErrIfNotFound = true, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } +#else /// public async Task ReadFileAsync( string index, @@ -113,4 +125,5 @@ public async Task ReadFileAsync( throw new ContentStorageFileNotFoundException("File not found"); } } +#endif } diff --git a/service/Core/MemoryServerless.cs b/service/Core/MemoryServerless.cs index b5b88b751..762dbb99b 100644 --- a/service/Core/MemoryServerless.cs +++ b/service/Core/MemoryServerless.cs @@ -191,6 +191,14 @@ public async Task IsDocumentReadyAsync( } } +#if KernelMemoryDev + /// + public Task ExportFileAsync(string documentId, string fileName, string? index = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } +#endif + /// public Task SearchAsync( string query, diff --git a/service/Core/MemoryService.cs b/service/Core/MemoryService.cs index 7f760eb7e..5a6c35268 100644 --- a/service/Core/MemoryService.cs +++ b/service/Core/MemoryService.cs @@ -168,6 +168,14 @@ public Task IsDocumentReadyAsync( return this._orchestrator.ReadPipelineSummaryAsync(index: index, documentId, cancellationToken); } +#if KernelMemoryDev + /// + public Task ExportFileAsync(string documentId, string fileName, string? index = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } +#endif + /// public Task SearchAsync( string query, diff --git a/service/Core/Pipeline/BaseOrchestrator.cs b/service/Core/Pipeline/BaseOrchestrator.cs index d187f5d15..5fab95c3e 100644 --- a/service/Core/Pipeline/BaseOrchestrator.cs +++ b/service/Core/Pipeline/BaseOrchestrator.cs @@ -1,5 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. +// ReSharper disable RedundantUsingDirective +#pragma warning disable CS0162 // temp +#pragma warning disable CS1998 // temp +#pragma warning disable IDE0005 // temp + using System; using System.Collections.Generic; using System.Linq; @@ -147,6 +152,9 @@ public DataPipeline PrepareNewDocumentUpload( /// public async Task ReadPipelineStatusAsync(string index, string documentId, CancellationToken cancellationToken = default) { +#if KernelMemoryDev + throw new NotImplementedException(); +#else index = IndexName.CleanName(index, this._defaultIndexName); try @@ -172,6 +180,7 @@ public DataPipeline PrepareNewDocumentUpload( { throw new PipelineNotFoundException("Pipeline/Document not found"); } +#endif } /// @@ -212,6 +221,13 @@ public Task StopAllPipelinesAsync() return this.CancellationTokenSource.CancelAsync(); } +#if KernelMemoryDev + public Task ReadFileAsStreamAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } +#endif + /// public async Task ReadTextFileAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default) { @@ -223,7 +239,11 @@ public async Task ReadTextFileAsync(DataPipeline pipeline, string fileNa public Task ReadFileAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default) { pipeline.Index = IndexName.CleanName(pipeline.Index, this._defaultIndexName); +#if KernelMemoryDev + throw new NotImplementedException(); +#else return this._contentStorage.ReadFileAsync(pipeline.Index, pipeline.DocumentId, fileName, true, cancellationToken); +#endif } /// diff --git a/tools/upload-file.sh b/tools/upload-file.sh index c0c12687c..1a1c5f06f 100755 --- a/tools/upload-file.sh +++ b/tools/upload-file.sh @@ -27,7 +27,7 @@ Usage: Example: - ./upload-file.sh -s http://127.0.0.1:9001 -f myFile.pdf -u me -t "type:notes" -t "type:test" -i "bash test" + ./upload-file.sh -s http://127.0.0.1:9001 -f myFile.pdf -p me -t "type:notes" -t "type:test" -i "bash test" For more information visit https://github.com/microsoft/kernel-memory @@ -69,11 +69,11 @@ readParameters() { validateParameters() { if [ -z "$SERVICE_URL" ]; then - echo "Please specify the web service URL" + echo "Please specify the web service URL. Use -h option for instructions." exit 1 fi if [ -z "$FILENAME" ]; then - echo "Please specify a file to upload" + echo "Please specify a file to upload. Use -h option for instructions." exit 3 fi if [ -d "$FILENAME" ]; then