New abstractions to support file download (#448)

## Motivation and Context (Why the change? What's the scenario?)

Supporting abstractions for new File Download feature. See also PR #415

## High level description (Approach, Design)

* New version 0.40
* Breaking changes on storage interface
* New methods on orchestration interface
* New methods on memory interface

KernelMemory.sln.DotSettings

@@ -271,6 +271,7 @@ public void It$SOMENAME$()
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=skprompt/@EntryIndexedValue">True</s:Boolean>
 	<s:String x:Key="/Default/CodeInspection/Highlighting/InspectionSeverities/=Xunit_002EXunitTestWithConsoleOutput/@EntryIndexedValue">DO_NOT_SHOW</s:String>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=smemory/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=subdir/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=SVCS/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=syntaxes/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=testsettings/@EntryIndexedValue">True</s:Boolean>

clients/dotnet/WebClient/MemoryWebClient.cs

@@ -233,6 +233,13 @@ public async Task<bool> IsDocumentReadyAsync(
         return status;
     }
 
+#if KernelMemoryDev
+    public Task<StreamableFileContent> ExportFileAsync(string documentId, string fileName, string? index = null, CancellationToken cancellationToken = default)
+    {
+        throw new NotImplementedException();
+    }
+#endif
+
     /// <inheritdoc />
     public async Task<SearchResult> SearchAsync(
         string query,

clients/dotnet/WebClient/WebClient.csproj

@@ -5,6 +5,7 @@
         <AssemblyName>Microsoft.KernelMemory.WebClient</AssemblyName>
         <RootNamespace>Microsoft.KernelMemory</RootNamespace>
         <NoWarn>$(NoWarn);CS1591;NU5104;</NoWarn>
+        <DefineConstants Condition="'$(SolutionName)' == 'KernelMemoryDev'">$(DefineConstants);KernelMemoryDev</DefineConstants>
     </PropertyGroup>
 
     <ItemGroup>

examples/006-curl-calling-webservice/README.md

@@ -18,7 +18,7 @@ Content of [upload-example.sh](upload-example.sh):
 ```bash
 ../../tools/upload-file.sh -f test.pdf \
                            -s http://127.0.0.1:9001 \
-                           -u curlUser \
+                           -p curlUser \
                            -t "type:test" \
                            -i curlExample01
 ```
@@ -33,7 +33,7 @@ Content of [ask-example.sh](ask-example.sh):
 
 ```bash
 ../../tools/ask.sh -s http://127.0.0.1:9001 \
-                   -u curlUser \
+                   -p curlUser \
                    -q "tell me about Semantic Kernel" \
                    -f '"type":["test"]'
 ```

extensions/AzureBlobs/AzureBlobs.csproj

@@ -6,6 +6,7 @@
         <AssemblyName>Microsoft.KernelMemory.ContentStorage.AzureBlobs</AssemblyName>
         <RootNamespace>Microsoft.KernelMemory.ContentStorage.AzureBlobs</RootNamespace>
         <NoWarn>$(NoWarn);CA1724;CS1591;</NoWarn>
+        <DefineConstants Condition="'$(SolutionName)' == 'KernelMemoryDev'">$(DefineConstants);KernelMemoryDev</DefineConstants>
     </PropertyGroup>
 
     <ItemGroup>

extensions/AzureBlobs/AzureBlobsStorage.cs

@@ -189,6 +189,18 @@ public Task WriteFileAsync(
         return this.InternalWriteAsync(directoryName, fileName, streamContent, cancellationToken);
     }
 
+#if KernelMemoryDev
+    /// <inheritdoc />
+    public Task<StreamableFileContent> ReadFileAsync(
+        string index,
+        string documentId,
+        string fileName,
+        bool logErrIfNotFound = true,
+        CancellationToken cancellationToken = default)
+    {
+        throw new NotImplementedException();
+    }
+#else
     /// <inheritdoc />
     public async Task<BinaryData> ReadFileAsync(
         string index,
@@ -220,6 +232,7 @@ public async Task<BinaryData> ReadFileAsync(
             throw new ContentStorageFileNotFoundException("File not found", e);
         }
     }
+#endif
 
     #region private
 

extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlas.csproj

@@ -6,6 +6,7 @@
         <AssemblyName>Microsoft.KernelMemory.MongoDbAtlas</AssemblyName>
         <RootNamespace>Microsoft.KernelMemory.MongoDbAtlas</RootNamespace>
         <NoWarn>$(NoWarn);CA1724;CS1591;CA1308;</NoWarn>
+        <DefineConstants Condition="'$(SolutionName)' == 'KernelMemoryDev'">$(DefineConstants);KernelMemoryDev</DefineConstants>
     </PropertyGroup>
 
     <ItemGroup>

extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs

@@ -124,6 +124,19 @@ public Task CreateDocumentDirectoryAsync(string index, string documentId,
         return Task.CompletedTask;
     }
 
+#if KernelMemoryDev
+    /// <inheritdoc />
+    public Task<StreamableFileContent> ReadFileAsync(
+        string index,
+        string documentId,
+        string fileName,
+        bool logErrIfNotFound = true,
+        CancellationToken cancellationToken = default)
+    {
+        throw new NotImplementedException();
+    }
+#else
+    /// <inheritdoc />
     public async Task<BinaryData> ReadFileAsync(string index, string documentId, string fileName, bool logErrIfNotFound = true,
         CancellationToken cancellationToken = new CancellationToken())
     {
@@ -189,6 +202,7 @@ public async Task<BinaryData> ReadFileAsync(string index, string documentId, str
             return new BinaryData(memoryStream.ToArray());
         }
     }
+#endif
 
     private async Task SaveDocumentAsync(string index, string id, BsonDocument doc, CancellationToken cancellationToken)
     {

extensions/TikToken/TikToken/TikToken.csproj

@@ -5,7 +5,7 @@
         <RollForward>LatestMajor</RollForward>
         <AssemblyName>Microsoft.KernelMemory.AI.TikToken</AssemblyName>
         <RootNamespace>Microsoft.KernelMemory.AI.TikToken</RootNamespace>
-        <NoWarn>$(NoWarn);</NoWarn>
+        <NoWarn>$(NoWarn);NU5104;</NoWarn>
     </PropertyGroup>
 
     <ItemGroup>

nuget-package.props

@@ -1,7 +1,7 @@
 <Project>
     <PropertyGroup>
         <!-- Central version prefix - applies to all nuget packages. -->
-        <Version>0.39.0</Version>
+        <Version>0.40.0</Version>
 
         <!-- These are set at the project level-->
         <IsPackable>false</IsPackable>

service/Abstractions/Constants.cs

@@ -13,6 +13,9 @@ public static class Constants
     // Form field containing the Document ID
     public const string WebServiceDocumentIdField = "documentId";
 
+    // Form field containing the Filename
+    public const string WebServiceFilenameField = "filename";
+
     // Form field containing the list of tags
     public const string WebServiceTagsField = "tags";
 
@@ -50,6 +53,7 @@ public static class Constants
     // Endpoints
     public const string HttpAskEndpoint = "/ask";
     public const string HttpSearchEndpoint = "/search";
+    public const string HttpDownloadEndpoint = "/download";
     public const string HttpUploadEndpoint = "/upload";
     public const string HttpUploadStatusEndpoint = "/upload-status";
     public const string HttpDocumentsEndpoint = "/documents";

service/Abstractions/ContentStorage/IContentStorage.cs

@@ -1,6 +1,5 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using System;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
@@ -84,7 +83,7 @@ Task WriteFileAsync(
     /// <param name="logErrIfNotFound">Whether to log an error if the file does not exist. An exception will be raised anyway.</param>
     /// <param name="cancellationToken">Async task cancellation token</param>
     /// <returns>File content</returns>
-    Task<BinaryData> ReadFileAsync(
+    Task<StreamableFileContent> ReadFileAsync(
         string index,
         string documentId,
         string fileName,

service/Abstractions/IKernelMemory.cs

@@ -160,6 +160,20 @@ public Task<bool> IsDocumentReadyAsync(
         string? index = null,
         CancellationToken cancellationToken = default);
 
+    /// <summary>
+    /// Export a file from content storage
+    /// </summary>
+    /// <param name="documentId">ID of the document containing the file</param>
+    /// <param name="fileName">File name</param>
+    /// <param name="index">Index containing the document</param>
+    /// <param name="cancellationToken">Async task cancellation token</param>
+    /// <returns>File content</returns>
+    public Task<StreamableFileContent> ExportFileAsync(
+        string documentId,
+        string fileName,
+        string? index = null,
+        CancellationToken cancellationToken = default);
+
     /// <summary>
     /// Search the given index for a list of relevant documents for the given query.
     /// </summary>

service/Abstractions/Models/StreamableFileContent.cs

@@ -0,0 +1,53 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Microsoft.KernelMemory;
+
+public sealed class StreamableFileContent : IDisposable
+{
+    private Stream? _stream;
+
+    public string FileName { get; } = string.Empty;
+    public long FileSize { get; } = 0;
+    public string FileType { get; } = string.Empty;
+    public DateTimeOffset LastWrite { get; } = default;
+    public Func<Task<Stream>> StreamAsync { get; }
+
+    public StreamableFileContent()
+    {
+        this.StreamAsync = () => Task.FromResult<Stream>(new MemoryStream());
+    }
+
+    public StreamableFileContent(
+        string fileName,
+        long fileSize,
+        string fileType = "application/octet-stream",
+        DateTimeOffset lastWriteTimeUtc = default,
+        Func<Task<Stream>>? asyncStreamDelegate = null)
+    {
+        ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(fileType, nameof(fileType), "File content type is empty");
+        ArgumentNullExceptionEx.ThrowIfNull(lastWriteTimeUtc, nameof(lastWriteTimeUtc), "File last write time is NULL");
+        ArgumentNullExceptionEx.ThrowIfNull(asyncStreamDelegate, nameof(asyncStreamDelegate), "asyncStreamDelegate is NULL");
+
+        this.FileName = fileName;
+        this.FileSize = fileSize;
+        this.FileType = fileType;
+        this.LastWrite = lastWriteTimeUtc;
+        this.StreamAsync = async () =>
+        {
+            this._stream = await asyncStreamDelegate().ConfigureAwait(false);
+            return this._stream;
+        };
+    }
+
+    public void Dispose()
+    {
+        if (this._stream == null) { return; }
+
+        this._stream.Close();
+        this._stream.Dispose();
+    }
+}

service/Abstractions/Pipeline/IPipelineOrchestrator.cs

@@ -91,6 +91,15 @@ public interface IPipelineOrchestrator
     /// </summary>
     Task StopAllPipelinesAsync();
 
+    /// <summary>
+    /// Fetch a file from content storage, streaming its content and details
+    /// </summary>
+    /// <param name="pipeline">Pipeline containing the file</param>
+    /// <param name="fileName">Name of the file to fetch</param>
+    /// <param name="cancellationToken">Async task cancellation token</param>
+    /// <returns>File data</returns>
+    Task<StreamableFileContent> ReadFileAsStreamAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default);
+
     /// <summary>
     /// Fetch a file from content storage
     /// </summary>

service/Core/ContentStorage/DevTools/SimpleFileStorage.cs

@@ -91,6 +91,18 @@ public async Task WriteFileAsync(
         await this._fileSystem.WriteFileAsync(volume: index, relPath: documentId, fileName: fileName, streamContent: streamContent, cancellationToken).ConfigureAwait(false);
     }
 
+#if KernelMemoryDev
+    /// <inheritdoc />
+    public Task<StreamableFileContent> ReadFileAsync(
+        string index,
+        string documentId,
+        string fileName,
+        bool logErrIfNotFound = true,
+        CancellationToken cancellationToken = default)
+    {
+        throw new NotImplementedException();
+    }
+#else
     /// <inheritdoc />
     public async Task<BinaryData> ReadFileAsync(
         string index,
@@ -113,4 +125,5 @@ public async Task<BinaryData> ReadFileAsync(
             throw new ContentStorageFileNotFoundException("File not found");
         }
     }
+#endif
 }

service/Core/MemoryServerless.cs

@@ -191,6 +191,14 @@ public async Task<bool> IsDocumentReadyAsync(
         }
     }
 
+#if KernelMemoryDev
+    /// <inheritdoc />
+    public Task<StreamableFileContent> ExportFileAsync(string documentId, string fileName, string? index = null, CancellationToken cancellationToken = default)
+    {
+        throw new NotImplementedException();
+    }
+#endif
+
     /// <inheritdoc />
     public Task<SearchResult> SearchAsync(
         string query,

service/Core/MemoryService.cs

@@ -168,6 +168,14 @@ public Task<bool> IsDocumentReadyAsync(
         return this._orchestrator.ReadPipelineSummaryAsync(index: index, documentId, cancellationToken);
     }
 
+#if KernelMemoryDev
+    /// <inheritdoc />
+    public Task<StreamableFileContent> ExportFileAsync(string documentId, string fileName, string? index = null, CancellationToken cancellationToken = default)
+    {
+        throw new NotImplementedException();
+    }
+#endif
+
     /// <inheritdoc />
     public Task<SearchResult> SearchAsync(
         string query,

service/Core/Pipeline/BaseOrchestrator.cs

@@ -1,5 +1,10 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+// ReSharper disable RedundantUsingDirective
+#pragma warning disable CS0162 // temp
+#pragma warning disable CS1998 // temp
+#pragma warning disable IDE0005 // temp
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -147,6 +152,9 @@ public DataPipeline PrepareNewDocumentUpload(
     ///<inheritdoc />
     public async Task<DataPipeline?> ReadPipelineStatusAsync(string index, string documentId, CancellationToken cancellationToken = default)
     {
+#if KernelMemoryDev
+        throw new NotImplementedException();
+#else
         index = IndexName.CleanName(index, this._defaultIndexName);
 
         try
@@ -172,6 +180,7 @@ public DataPipeline PrepareNewDocumentUpload(
         {
             throw new PipelineNotFoundException("Pipeline/Document not found");
         }
+#endif
     }
 
     ///<inheritdoc />
@@ -212,6 +221,13 @@ public Task StopAllPipelinesAsync()
         return this.CancellationTokenSource.CancelAsync();
     }
 
+#if KernelMemoryDev
+    public Task<StreamableFileContent> ReadFileAsStreamAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default)
+    {
+        throw new NotImplementedException();
+    }
+#endif
+
     ///<inheritdoc />
     public async Task<string> ReadTextFileAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default)
     {
@@ -223,7 +239,11 @@ public async Task<string> ReadTextFileAsync(DataPipeline pipeline, string fileNa
     public Task<BinaryData> ReadFileAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default)
     {
         pipeline.Index = IndexName.CleanName(pipeline.Index, this._defaultIndexName);
+#if KernelMemoryDev
+        throw new NotImplementedException();
+#else
         return this._contentStorage.ReadFileAsync(pipeline.Index, pipeline.DocumentId, fileName, true, cancellationToken);
+#endif
     }
 
     ///<inheritdoc />

tools/upload-file.sh

@@ -27,7 +27,7 @@ Usage:
 
 Example:
 
-    ./upload-file.sh -s http://127.0.0.1:9001 -f myFile.pdf -u me -t "type:notes" -t "type:test" -i "bash test"
+    ./upload-file.sh -s http://127.0.0.1:9001 -f myFile.pdf -p me -t "type:notes" -t "type:test" -i "bash test"
 
 
 For more information visit https://github.com/microsoft/kernel-memory
@@ -69,11 +69,11 @@ readParameters() {
 
 validateParameters() {
     if [ -z "$SERVICE_URL" ]; then
-        echo "Please specify the web service URL"
+        echo "Please specify the web service URL. Use -h option for instructions."
         exit 1
     fi
     if [ -z "$FILENAME" ]; then
-        echo "Please specify a file to upload"
+        echo "Please specify a file to upload. Use -h option for instructions."
         exit 3
     fi
     if [ -d "$FILENAME" ]; then