diff --git a/KernelMemory.sln.DotSettings b/KernelMemory.sln.DotSettings
index d9a10ac0b..cc1462402 100644
--- a/KernelMemory.sln.DotSettings
+++ b/KernelMemory.sln.DotSettings
@@ -271,6 +271,7 @@ public void It$SOMENAME$()
True
DO_NOT_SHOW
True
+ True
True
True
True
diff --git a/clients/dotnet/WebClient/MemoryWebClient.cs b/clients/dotnet/WebClient/MemoryWebClient.cs
index 730968374..2e60cb343 100644
--- a/clients/dotnet/WebClient/MemoryWebClient.cs
+++ b/clients/dotnet/WebClient/MemoryWebClient.cs
@@ -233,6 +233,13 @@ public async Task IsDocumentReadyAsync(
return status;
}
+#if KernelMemoryDev
+ public Task ExportFileAsync(string documentId, string fileName, string? index = null, CancellationToken cancellationToken = default)
+ {
+ throw new NotImplementedException();
+ }
+#endif
+
///
public async Task SearchAsync(
string query,
diff --git a/clients/dotnet/WebClient/WebClient.csproj b/clients/dotnet/WebClient/WebClient.csproj
index b69d40942..2f673a08d 100644
--- a/clients/dotnet/WebClient/WebClient.csproj
+++ b/clients/dotnet/WebClient/WebClient.csproj
@@ -5,6 +5,7 @@
Microsoft.KernelMemory.WebClient
Microsoft.KernelMemory
$(NoWarn);CS1591;NU5104;
+ $(DefineConstants);KernelMemoryDev
diff --git a/examples/006-curl-calling-webservice/README.md b/examples/006-curl-calling-webservice/README.md
index ac82f947a..48a5762ff 100644
--- a/examples/006-curl-calling-webservice/README.md
+++ b/examples/006-curl-calling-webservice/README.md
@@ -18,7 +18,7 @@ Content of [upload-example.sh](upload-example.sh):
```bash
../../tools/upload-file.sh -f test.pdf \
-s http://127.0.0.1:9001 \
- -u curlUser \
+ -p curlUser \
-t "type:test" \
-i curlExample01
```
@@ -33,7 +33,7 @@ Content of [ask-example.sh](ask-example.sh):
```bash
../../tools/ask.sh -s http://127.0.0.1:9001 \
- -u curlUser \
+ -p curlUser \
-q "tell me about Semantic Kernel" \
-f '"type":["test"]'
```
\ No newline at end of file
diff --git a/extensions/AzureBlobs/AzureBlobs.csproj b/extensions/AzureBlobs/AzureBlobs.csproj
index a3786f6cf..35ca67431 100644
--- a/extensions/AzureBlobs/AzureBlobs.csproj
+++ b/extensions/AzureBlobs/AzureBlobs.csproj
@@ -6,6 +6,7 @@
Microsoft.KernelMemory.ContentStorage.AzureBlobs
Microsoft.KernelMemory.ContentStorage.AzureBlobs
$(NoWarn);CA1724;CS1591;
+ $(DefineConstants);KernelMemoryDev
diff --git a/extensions/AzureBlobs/AzureBlobsStorage.cs b/extensions/AzureBlobs/AzureBlobsStorage.cs
index 0a81579a9..e1ee29b62 100644
--- a/extensions/AzureBlobs/AzureBlobsStorage.cs
+++ b/extensions/AzureBlobs/AzureBlobsStorage.cs
@@ -189,6 +189,18 @@ public Task WriteFileAsync(
return this.InternalWriteAsync(directoryName, fileName, streamContent, cancellationToken);
}
+#if KernelMemoryDev
+ ///
+ public Task ReadFileAsync(
+ string index,
+ string documentId,
+ string fileName,
+ bool logErrIfNotFound = true,
+ CancellationToken cancellationToken = default)
+ {
+ throw new NotImplementedException();
+ }
+#else
///
public async Task ReadFileAsync(
string index,
@@ -220,6 +232,7 @@ public async Task ReadFileAsync(
throw new ContentStorageFileNotFoundException("File not found", e);
}
}
+#endif
#region private
diff --git a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlas.csproj b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlas.csproj
index b5f9a62fd..78f42b5a8 100644
--- a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlas.csproj
+++ b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlas.csproj
@@ -6,6 +6,7 @@
Microsoft.KernelMemory.MongoDbAtlas
Microsoft.KernelMemory.MongoDbAtlas
$(NoWarn);CA1724;CS1591;CA1308;
+ $(DefineConstants);KernelMemoryDev
diff --git a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs
index f245366f7..f9b4a7fab 100644
--- a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs
+++ b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs
@@ -124,6 +124,19 @@ public Task CreateDocumentDirectoryAsync(string index, string documentId,
return Task.CompletedTask;
}
+#if KernelMemoryDev
+ ///
+ public Task ReadFileAsync(
+ string index,
+ string documentId,
+ string fileName,
+ bool logErrIfNotFound = true,
+ CancellationToken cancellationToken = default)
+ {
+ throw new NotImplementedException();
+ }
+#else
+ ///
public async Task ReadFileAsync(string index, string documentId, string fileName, bool logErrIfNotFound = true,
CancellationToken cancellationToken = new CancellationToken())
{
@@ -189,6 +202,7 @@ public async Task ReadFileAsync(string index, string documentId, str
return new BinaryData(memoryStream.ToArray());
}
}
+#endif
private async Task SaveDocumentAsync(string index, string id, BsonDocument doc, CancellationToken cancellationToken)
{
diff --git a/extensions/TikToken/TikToken/TikToken.csproj b/extensions/TikToken/TikToken/TikToken.csproj
index 735ccdd47..7bfc04884 100644
--- a/extensions/TikToken/TikToken/TikToken.csproj
+++ b/extensions/TikToken/TikToken/TikToken.csproj
@@ -5,7 +5,7 @@
LatestMajor
Microsoft.KernelMemory.AI.TikToken
Microsoft.KernelMemory.AI.TikToken
- $(NoWarn);
+ $(NoWarn);NU5104;
diff --git a/nuget-package.props b/nuget-package.props
index 4c182da8d..3d7ade528 100644
--- a/nuget-package.props
+++ b/nuget-package.props
@@ -1,7 +1,7 @@
- 0.39.0
+ 0.40.0
false
diff --git a/service/Abstractions/Constants.cs b/service/Abstractions/Constants.cs
index 5dfb4ad2e..b9a13a5f3 100644
--- a/service/Abstractions/Constants.cs
+++ b/service/Abstractions/Constants.cs
@@ -13,6 +13,9 @@ public static class Constants
// Form field containing the Document ID
public const string WebServiceDocumentIdField = "documentId";
+ // Form field containing the Filename
+ public const string WebServiceFilenameField = "filename";
+
// Form field containing the list of tags
public const string WebServiceTagsField = "tags";
@@ -50,6 +53,7 @@ public static class Constants
// Endpoints
public const string HttpAskEndpoint = "/ask";
public const string HttpSearchEndpoint = "/search";
+ public const string HttpDownloadEndpoint = "/download";
public const string HttpUploadEndpoint = "/upload";
public const string HttpUploadStatusEndpoint = "/upload-status";
public const string HttpDocumentsEndpoint = "/documents";
diff --git a/service/Abstractions/ContentStorage/IContentStorage.cs b/service/Abstractions/ContentStorage/IContentStorage.cs
index 7e3f9e6d1..ab93b0d0d 100644
--- a/service/Abstractions/ContentStorage/IContentStorage.cs
+++ b/service/Abstractions/ContentStorage/IContentStorage.cs
@@ -1,6 +1,5 @@
// Copyright (c) Microsoft. All rights reserved.
-using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
@@ -84,7 +83,7 @@ Task WriteFileAsync(
/// Whether to log an error if the file does not exist. An exception will be raised anyway.
/// Async task cancellation token
/// File content
- Task ReadFileAsync(
+ Task ReadFileAsync(
string index,
string documentId,
string fileName,
diff --git a/service/Abstractions/IKernelMemory.cs b/service/Abstractions/IKernelMemory.cs
index a2b0c8d7e..bf7bed55d 100644
--- a/service/Abstractions/IKernelMemory.cs
+++ b/service/Abstractions/IKernelMemory.cs
@@ -160,6 +160,20 @@ public Task IsDocumentReadyAsync(
string? index = null,
CancellationToken cancellationToken = default);
+ ///
+ /// Export a file from content storage
+ ///
+ /// ID of the document containing the file
+ /// File name
+ /// Index containing the document
+ /// Async task cancellation token
+ /// File content
+ public Task ExportFileAsync(
+ string documentId,
+ string fileName,
+ string? index = null,
+ CancellationToken cancellationToken = default);
+
///
/// Search the given index for a list of relevant documents for the given query.
///
diff --git a/service/Abstractions/Models/StreamableFileContent.cs b/service/Abstractions/Models/StreamableFileContent.cs
new file mode 100644
index 000000000..60e5ae1ef
--- /dev/null
+++ b/service/Abstractions/Models/StreamableFileContent.cs
@@ -0,0 +1,53 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Microsoft.KernelMemory;
+
+public sealed class StreamableFileContent : IDisposable
+{
+ private Stream? _stream;
+
+ public string FileName { get; } = string.Empty;
+ public long FileSize { get; } = 0;
+ public string FileType { get; } = string.Empty;
+ public DateTimeOffset LastWrite { get; } = default;
+ public Func> StreamAsync { get; }
+
+ public StreamableFileContent()
+ {
+ this.StreamAsync = () => Task.FromResult(new MemoryStream());
+ }
+
+ public StreamableFileContent(
+ string fileName,
+ long fileSize,
+ string fileType = "application/octet-stream",
+ DateTimeOffset lastWriteTimeUtc = default,
+ Func>? asyncStreamDelegate = null)
+ {
+ ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(fileType, nameof(fileType), "File content type is empty");
+ ArgumentNullExceptionEx.ThrowIfNull(lastWriteTimeUtc, nameof(lastWriteTimeUtc), "File last write time is NULL");
+ ArgumentNullExceptionEx.ThrowIfNull(asyncStreamDelegate, nameof(asyncStreamDelegate), "asyncStreamDelegate is NULL");
+
+ this.FileName = fileName;
+ this.FileSize = fileSize;
+ this.FileType = fileType;
+ this.LastWrite = lastWriteTimeUtc;
+ this.StreamAsync = async () =>
+ {
+ this._stream = await asyncStreamDelegate().ConfigureAwait(false);
+ return this._stream;
+ };
+ }
+
+ public void Dispose()
+ {
+ if (this._stream == null) { return; }
+
+ this._stream.Close();
+ this._stream.Dispose();
+ }
+}
diff --git a/service/Abstractions/Pipeline/IPipelineOrchestrator.cs b/service/Abstractions/Pipeline/IPipelineOrchestrator.cs
index 12cb85739..52ff6d8a3 100644
--- a/service/Abstractions/Pipeline/IPipelineOrchestrator.cs
+++ b/service/Abstractions/Pipeline/IPipelineOrchestrator.cs
@@ -91,6 +91,15 @@ public interface IPipelineOrchestrator
///
Task StopAllPipelinesAsync();
+ ///
+ /// Fetch a file from content storage, streaming its content and details
+ ///
+ /// Pipeline containing the file
+ /// Name of the file to fetch
+ /// Async task cancellation token
+ /// File data
+ Task ReadFileAsStreamAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default);
+
///
/// Fetch a file from content storage
///
diff --git a/service/Core/ContentStorage/DevTools/SimpleFileStorage.cs b/service/Core/ContentStorage/DevTools/SimpleFileStorage.cs
index 89c9a3e43..69484f051 100644
--- a/service/Core/ContentStorage/DevTools/SimpleFileStorage.cs
+++ b/service/Core/ContentStorage/DevTools/SimpleFileStorage.cs
@@ -91,6 +91,18 @@ public async Task WriteFileAsync(
await this._fileSystem.WriteFileAsync(volume: index, relPath: documentId, fileName: fileName, streamContent: streamContent, cancellationToken).ConfigureAwait(false);
}
+#if KernelMemoryDev
+ ///
+ public Task ReadFileAsync(
+ string index,
+ string documentId,
+ string fileName,
+ bool logErrIfNotFound = true,
+ CancellationToken cancellationToken = default)
+ {
+ throw new NotImplementedException();
+ }
+#else
///
public async Task ReadFileAsync(
string index,
@@ -113,4 +125,5 @@ public async Task ReadFileAsync(
throw new ContentStorageFileNotFoundException("File not found");
}
}
+#endif
}
diff --git a/service/Core/MemoryServerless.cs b/service/Core/MemoryServerless.cs
index b5b88b751..762dbb99b 100644
--- a/service/Core/MemoryServerless.cs
+++ b/service/Core/MemoryServerless.cs
@@ -191,6 +191,14 @@ public async Task IsDocumentReadyAsync(
}
}
+#if KernelMemoryDev
+ ///
+ public Task ExportFileAsync(string documentId, string fileName, string? index = null, CancellationToken cancellationToken = default)
+ {
+ throw new NotImplementedException();
+ }
+#endif
+
///
public Task SearchAsync(
string query,
diff --git a/service/Core/MemoryService.cs b/service/Core/MemoryService.cs
index 7f760eb7e..5a6c35268 100644
--- a/service/Core/MemoryService.cs
+++ b/service/Core/MemoryService.cs
@@ -168,6 +168,14 @@ public Task IsDocumentReadyAsync(
return this._orchestrator.ReadPipelineSummaryAsync(index: index, documentId, cancellationToken);
}
+#if KernelMemoryDev
+ ///
+ public Task ExportFileAsync(string documentId, string fileName, string? index = null, CancellationToken cancellationToken = default)
+ {
+ throw new NotImplementedException();
+ }
+#endif
+
///
public Task SearchAsync(
string query,
diff --git a/service/Core/Pipeline/BaseOrchestrator.cs b/service/Core/Pipeline/BaseOrchestrator.cs
index d187f5d15..5fab95c3e 100644
--- a/service/Core/Pipeline/BaseOrchestrator.cs
+++ b/service/Core/Pipeline/BaseOrchestrator.cs
@@ -1,5 +1,10 @@
// Copyright (c) Microsoft. All rights reserved.
+// ReSharper disable RedundantUsingDirective
+#pragma warning disable CS0162 // temp
+#pragma warning disable CS1998 // temp
+#pragma warning disable IDE0005 // temp
+
using System;
using System.Collections.Generic;
using System.Linq;
@@ -147,6 +152,9 @@ public DataPipeline PrepareNewDocumentUpload(
///
public async Task ReadPipelineStatusAsync(string index, string documentId, CancellationToken cancellationToken = default)
{
+#if KernelMemoryDev
+ throw new NotImplementedException();
+#else
index = IndexName.CleanName(index, this._defaultIndexName);
try
@@ -172,6 +180,7 @@ public DataPipeline PrepareNewDocumentUpload(
{
throw new PipelineNotFoundException("Pipeline/Document not found");
}
+#endif
}
///
@@ -212,6 +221,13 @@ public Task StopAllPipelinesAsync()
return this.CancellationTokenSource.CancelAsync();
}
+#if KernelMemoryDev
+ public Task ReadFileAsStreamAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default)
+ {
+ throw new NotImplementedException();
+ }
+#endif
+
///
public async Task ReadTextFileAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default)
{
@@ -223,7 +239,11 @@ public async Task ReadTextFileAsync(DataPipeline pipeline, string fileNa
public Task ReadFileAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default)
{
pipeline.Index = IndexName.CleanName(pipeline.Index, this._defaultIndexName);
+#if KernelMemoryDev
+ throw new NotImplementedException();
+#else
return this._contentStorage.ReadFileAsync(pipeline.Index, pipeline.DocumentId, fileName, true, cancellationToken);
+#endif
}
///
diff --git a/tools/upload-file.sh b/tools/upload-file.sh
index c0c12687c..1a1c5f06f 100755
--- a/tools/upload-file.sh
+++ b/tools/upload-file.sh
@@ -27,7 +27,7 @@ Usage:
Example:
- ./upload-file.sh -s http://127.0.0.1:9001 -f myFile.pdf -u me -t "type:notes" -t "type:test" -i "bash test"
+ ./upload-file.sh -s http://127.0.0.1:9001 -f myFile.pdf -p me -t "type:notes" -t "type:test" -i "bash test"
For more information visit https://github.com/microsoft/kernel-memory
@@ -69,11 +69,11 @@ readParameters() {
validateParameters() {
if [ -z "$SERVICE_URL" ]; then
- echo "Please specify the web service URL"
+ echo "Please specify the web service URL. Use -h option for instructions."
exit 1
fi
if [ -z "$FILENAME" ]; then
- echo "Please specify a file to upload"
+ echo "Please specify a file to upload. Use -h option for instructions."
exit 3
fi
if [ -d "$FILENAME" ]; then