From 4aa666078e554d3a1b36ac4e0731417fbf5e4bc9 Mon Sep 17 00:00:00 2001 From: Devis Lucato Date: Wed, 23 Aug 2023 10:32:47 -0700 Subject: [PATCH] Add API allowing to store text/string, without the need to upload files --- dotnet/ClientLib/ISemanticMemoryClient.cs | 45 +++++++++++++++++++++-- dotnet/ClientLib/MemoryWebClient.cs | 33 ++++++++++++++++- dotnet/ClientLib/Models/Document.cs | 34 +++++++++++++++++ dotnet/CoreLib/Memory.cs | 38 ++++++++++++++++++- dotnet/CoreLib/MemoryService.cs | 38 ++++++++++++++++++- examples/001-dotnet-Serverless/Program.cs | 34 ++++++++++++++++- examples/002-dotnet-WebClient/Program.cs | 23 +++++++++++- 7 files changed, 234 insertions(+), 11 deletions(-) diff --git a/dotnet/ClientLib/ISemanticMemoryClient.cs b/dotnet/ClientLib/ISemanticMemoryClient.cs index 4dd84a26c..47edf7f06 100644 --- a/dotnet/ClientLib/ISemanticMemoryClient.cs +++ b/dotnet/ClientLib/ISemanticMemoryClient.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; @@ -24,9 +25,9 @@ public Task ImportDocumentAsync( CancellationToken cancellationToken = default); /// - /// Import a files from disk into memory, with details such as tags and user ID. + /// Import a file from disk into memory, with details such as tags and user ID. /// - /// Path and name of the files to import + /// Path and name of the file to import /// Document ID /// Optional tags to apply to the memories generated by the document /// Optional index name @@ -34,7 +35,7 @@ public Task ImportDocumentAsync( /// Async task cancellation token /// Document ID public Task ImportDocumentAsync( - string fileName, + string filePath, string? documentId = null, TagCollection? tags = null, string? index = null, @@ -65,6 +66,44 @@ public Task IsDocumentReadyAsync( string? index = null, CancellationToken cancellationToken = default); + /// + /// Import any stream from memory, e.g. text or binary data, with details such as tags and user ID. + /// + /// Content stream to import + /// File name to assign to the stream, used to detect the file type + /// Document ID + /// Optional tags to apply to the memories generated by the document + /// Optional index name + /// Ingestion pipeline steps, optional override to the system default + /// Async task cancellation token + /// Document ID + public Task ImportDocumentAsync( + Stream content, + string? fileName = null, + string? documentId = null, + TagCollection? tags = null, + string? index = null, + IEnumerable? steps = null, + CancellationToken cancellationToken = default); + + /// + /// Import any stream from memory, e.g. text or binary data, with details such as tags and user ID. + /// + /// Text content to import + /// Document ID + /// Optional tags to apply to the memories generated by the document + /// Optional index name + /// Ingestion pipeline steps, optional override to the system default + /// Async task cancellation token + /// Document ID + public Task ImportTextAsync( + string text, + string? documentId = null, + TagCollection? tags = null, + string? index = null, + IEnumerable? steps = null, + CancellationToken cancellationToken = default); + /// /// Get information about an uploaded document /// diff --git a/dotnet/ClientLib/MemoryWebClient.cs b/dotnet/ClientLib/MemoryWebClient.cs index 0d84f1e27..222aebe3a 100644 --- a/dotnet/ClientLib/MemoryWebClient.cs +++ b/dotnet/ClientLib/MemoryWebClient.cs @@ -40,14 +40,14 @@ public Task ImportDocumentAsync( /// public Task ImportDocumentAsync( - string fileName, + string filePath, string? documentId = null, TagCollection? tags = null, string? index = null, IEnumerable? steps = null, CancellationToken cancellationToken = default) { - var document = new Document(documentId, tags: tags).AddFile(fileName); + var document = new Document(documentId, tags: tags).AddFile(filePath); var uploadRequest = document.ToDocumentUploadRequest(index, steps); return this.ImportDocumentAsync(uploadRequest, cancellationToken); } @@ -61,6 +61,35 @@ public Task ImportDocumentAsync( return this.ImportInternalAsync(index, uploadRequest, cancellationToken); } + /// + public Task ImportDocumentAsync( + Stream content, + string? fileName = null, + string? documentId = null, + TagCollection? tags = null, + string? index = null, + IEnumerable? steps = null, + CancellationToken cancellationToken = default) + { + var document = new Document(documentId, tags: tags).AddStream(fileName, content); + var uploadRequest = document.ToDocumentUploadRequest(index, steps); + return this.ImportDocumentAsync(uploadRequest, cancellationToken); + } + + /// + public async Task ImportTextAsync( + string text, + string? documentId = null, + TagCollection? tags = null, + string? index = null, + IEnumerable? steps = null, + CancellationToken cancellationToken = default) + { + using Stream content = new MemoryStream(Encoding.UTF8.GetBytes(text)); + return await this.ImportDocumentAsync(content, fileName: "content.txt", documentId: documentId, tags: tags, index: index, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + /// public async Task IsDocumentReadyAsync( string documentId, diff --git a/dotnet/ClientLib/Models/Document.cs b/dotnet/ClientLib/Models/Document.cs index 2f569c9de..40e95bdf3 100644 --- a/dotnet/ClientLib/Models/Document.cs +++ b/dotnet/ClientLib/Models/Document.cs @@ -26,6 +26,8 @@ public string Id public List FileNames { get; set; } = new(); + public Dictionary Streams { get; set; } = new(); + public Document(string? id = null, TagCollection? tags = null, IEnumerable? fileNames = null) { if (id == null || string.IsNullOrWhiteSpace(id)) { id = RandomId(); } @@ -49,6 +51,32 @@ public Document AddFile(string fileName) return this; } + public Document AddStream(string? filename, Stream content) + { + if (content == null) + { + throw new SemanticMemoryException("The content stream is NULL"); + } + + if (string.IsNullOrWhiteSpace(filename)) + { + var i = 0; + filename = "content.txt"; + while (this.HasStream(filename)) + { + filename = $"content{i++}.txt"; + } + } + + this.Streams.Add(filename!, content); + return this; + } + + public bool HasStream(string name) + { + return this.Streams.ContainsKey(name); + } + public Document AddFiles(IEnumerable? fileNames) { if (fileNames != null) { this.FileNames.AddRange(fileNames); } @@ -106,6 +134,12 @@ public static DocumentUploadRequest ToDocumentUploadRequest( files.Add(formFile); } + foreach (KeyValuePair stream in doc.Streams) + { + var formFile = new DocumentUploadRequest.UploadedFile(stream.Key, stream.Value); + files.Add(formFile); + } + uploadRequest.Files = files; return uploadRequest; diff --git a/dotnet/CoreLib/Memory.cs b/dotnet/CoreLib/Memory.cs index 24789cc03..59b281494 100644 --- a/dotnet/CoreLib/Memory.cs +++ b/dotnet/CoreLib/Memory.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. using System.Collections.Generic; +using System.IO; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticMemory.Configuration; @@ -62,14 +64,14 @@ public async Task ImportDocumentAsync( /// public async Task ImportDocumentAsync( - string fileName, + string filePath, string? documentId = null, TagCollection? tags = null, string? index = null, IEnumerable? steps = null, CancellationToken cancellationToken = default) { - var document = new Document(documentId, tags: tags).AddFile(fileName); + var document = new Document(documentId, tags: tags).AddFile(filePath); var uploadRequest = await document.ToDocumentUploadRequestAsync(index, steps, cancellationToken).ConfigureAwait(false); return await this.ImportDocumentAsync(uploadRequest, cancellationToken).ConfigureAwait(false); } @@ -83,6 +85,38 @@ public async Task ImportDocumentAsync( return await this._orchestrator.ImportDocumentAsync(index, uploadRequest, cancellationToken).ConfigureAwait(false); } + /// + public async Task ImportDocumentAsync( + Stream content, + string? fileName = null, + string? documentId = null, + TagCollection? tags = null, + string? index = null, + IEnumerable? steps = null, + CancellationToken cancellationToken = default) + { + var document = new Document(documentId, tags: tags).AddStream(fileName, content); + var uploadRequest = await document.ToDocumentUploadRequestAsync(index, steps, cancellationToken).ConfigureAwait(false); + return await this.ImportDocumentAsync(uploadRequest, cancellationToken).ConfigureAwait(false); + } + + /// + public async Task ImportTextAsync( + string text, + string? documentId = null, + TagCollection? tags = null, + string? index = null, + IEnumerable? steps = null, + CancellationToken cancellationToken = default) + { + var content = new MemoryStream(Encoding.UTF8.GetBytes(text)); + await using (content.ConfigureAwait(false)) + { + return await this.ImportDocumentAsync(content, fileName: "content.txt", documentId: documentId, tags: tags, index: index, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + } + /// public async Task IsDocumentReadyAsync( string documentId, diff --git a/dotnet/CoreLib/MemoryService.cs b/dotnet/CoreLib/MemoryService.cs index 8aa7c0aae..e0501a347 100644 --- a/dotnet/CoreLib/MemoryService.cs +++ b/dotnet/CoreLib/MemoryService.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. using System.Collections.Generic; +using System.IO; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticMemory.Configuration; @@ -37,14 +39,14 @@ public async Task ImportDocumentAsync( /// public async Task ImportDocumentAsync( - string fileName, + string filePath, string? documentId = null, TagCollection? tags = null, string? index = null, IEnumerable? steps = null, CancellationToken cancellationToken = default) { - var document = new Document(documentId, tags: tags).AddFile(fileName); + var document = new Document(documentId, tags: tags).AddFile(filePath); var uploadRequest = await document.ToDocumentUploadRequestAsync(index, steps, cancellationToken).ConfigureAwait(false); return await this.ImportDocumentAsync(uploadRequest, cancellationToken).ConfigureAwait(false); } @@ -58,6 +60,38 @@ public Task ImportDocumentAsync( return this._orchestrator.ImportDocumentAsync(index, uploadRequest, cancellationToken); } + /// + public async Task ImportDocumentAsync( + Stream content, + string? fileName = null, + string? documentId = null, + TagCollection? tags = null, + string? index = null, + IEnumerable? steps = null, + CancellationToken cancellationToken = default) + { + var document = new Document(documentId, tags: tags).AddStream(fileName, content); + var uploadRequest = await document.ToDocumentUploadRequestAsync(index, steps, cancellationToken).ConfigureAwait(false); + return await this.ImportDocumentAsync(uploadRequest, cancellationToken).ConfigureAwait(false); + } + + /// + public async Task ImportTextAsync( + string text, + string? documentId = null, + TagCollection? tags = null, + string? index = null, + IEnumerable? steps = null, + CancellationToken cancellationToken = default) + { + var content = new MemoryStream(Encoding.UTF8.GetBytes(text)); + await using (content.ConfigureAwait(false)) + { + return await this.ImportDocumentAsync(content, fileName: "content.txt", documentId: documentId, tags: tags, index: index, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + } + /// public Task IsDocumentReadyAsync( string documentId, diff --git a/examples/001-dotnet-Serverless/Program.cs b/examples/001-dotnet-Serverless/Program.cs index ee6fbf0bc..f067a0484 100644 --- a/examples/001-dotnet-Serverless/Program.cs +++ b/examples/001-dotnet-Serverless/Program.cs @@ -21,6 +21,20 @@ // === UPLOAD ============ // ======================= +// Uploading some text, without using files +if (!await memory.IsDocumentReadyAsync(documentId: "doc000")) +{ + Console.WriteLine("Uploading doc000"); + await memory.ImportTextAsync("In physics, mass–energy equivalence is the relationship between mass and energy " + + "in a system's rest frame, where the two quantities differ only by a multiplicative " + + "constant and the units of measurement. The principle is described by the physicist " + + "Albert Einstein's formula: E = m*c^2", documentId: "doc000"); +} +else +{ + Console.WriteLine("doc000 already uploaded."); +} + // Simple file upload (checking if the file exists) if (!await memory.IsDocumentReadyAsync(documentId: "doc001")) { @@ -67,10 +81,17 @@ await memory.ImportDocumentAsync(new Document("doc003") // ======================= // Question without filters -var question = "What's Semantic Kernel?"; +var question = "What's mc^2?"; Console.WriteLine($"\n\nQuestion: {question}"); var answer = await memory.AskAsync(question); +Console.WriteLine($"\nAnswer: {answer.Result}\n\n"); + +// Another question without filters +question = "What's Semantic Kernel?"; +Console.WriteLine($"\n\nQuestion: {question}"); + +answer = await memory.AskAsync(question); Console.WriteLine($"\nAnswer: {answer.Result}\n\n Sources:\n"); foreach (var x in answer.RelevantSources) @@ -106,11 +127,22 @@ await memory.ImportDocumentAsync(new Document("doc003") // ReSharper disable CommentTypo /* ==== OUTPUT ==== +Uploading doc000 doc001 already uploaded. doc002 already uploaded. doc003 already uploaded. +Question: What's mc^2? + +Answer: mc^2 refers to the equation E=mc^2, which is the famous mass-energy equivalence equation proposed by +Albert Einstein in his theory of relativity. In this equation, E represents energy, m represents mass, and +c represents the speed of light in a vacuum. The equation states that energy is equal to mass multiplied by +the square of the speed of light. This equation shows the relationship between mass and energy, suggesting +that mass can be converted into energy and vice versa. It is a fundamental equation in physics and has +significant implications in various fields, including nuclear energy and particle physics. + + Question: What's Semantic Kernel? warn: Microsoft.SemanticMemory.Search.SearchClient[0] No memories available diff --git a/examples/002-dotnet-WebClient/Program.cs b/examples/002-dotnet-WebClient/Program.cs index ff5aa9d3e..cf9246f81 100644 --- a/examples/002-dotnet-WebClient/Program.cs +++ b/examples/002-dotnet-WebClient/Program.cs @@ -21,6 +21,20 @@ // Simple file upload (checking if the file exists) +// Uploading some text, without using files +if (!await memory.IsDocumentReadyAsync(documentId: "doc000")) +{ + Console.WriteLine("Uploading doc000"); + await memory.ImportTextAsync("In physics, mass–energy equivalence is the relationship between mass and energy " + + "in a system's rest frame, where the two quantities differ only by a multiplicative " + + "constant and the units of measurement. The principle is described by the physicist " + + "Albert Einstein's formula: E = m*c^2", documentId: "doc000"); +} +else +{ + Console.WriteLine("doc000 already uploaded."); +} + if (!await memory.IsDocumentReadyAsync(documentId: "doc001")) { Console.WriteLine("Uploading doc001"); @@ -76,10 +90,17 @@ await memory.ImportDocumentAsync(new Document("doc003") // ======================= // Question without filters -var question = "What's Semantic Kernel?"; +var question = "What's mc^2?"; Console.WriteLine($"\n\nQuestion: {question}"); var answer = await memory.AskAsync(question); +Console.WriteLine($"\nAnswer: {answer.Result}\n\n"); + +// Another question without filters +question = "What's Semantic Kernel?"; +Console.WriteLine($"\n\nQuestion: {question}"); + +answer = await memory.AskAsync(question); Console.WriteLine($"\nAnswer: {answer.Result}\n\n Sources:\n"); foreach (var x in answer.RelevantSources)