Upgrade packages, fix issue with OpenXml (#792)

Azure: * Azure.Storage.Blobs * Azure.Storage.Queues Document parsing: * ClosedXML 0.102.3 -> 0.104.0-rc1 * DocumentFormat.OpenXml 2.20.0 -> 3.1.0 SK: * Microsoft.SemanticKernel 1.18.2 -> 1.20.0 Misc: * AWSSDK * Microsoft.ML.Tokenizers * OllamaSharp * NRedisStack
microsoft · Sep 19, 2024 · 6c3689e · 6c3689e
1 parent 17cb2d2
commit 6c3689e
Show file tree

Hide file tree

Showing 8 changed files with 29 additions and 41 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -2,7 +2,7 @@
 <Project>
     <PropertyGroup>
         <!-- Central version prefix - applies to all nuget packages. -->
-        <Version>0.73.0</Version>
+        <Version>0.74.0</Version>
 
         <!-- C# lang version, https://learn.microsoft.com/dotnet/csharp/whats-new -->
         <LangVersion>12</LangVersion>

diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -4,15 +4,17 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageVersion Include="Aspire.Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.2.0" />
-    <PackageVersion Include="AWSSDK.S3" Version="3.7.402.7" />
+    <PackageVersion Include="AWSSDK.S3" Version="3.7.403" />
     <PackageVersion Include="Azure.AI.FormRecognizer" Version="4.1.0" />
     <PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.3.2" />
     <PackageVersion Include="Azure.Identity" Version="1.12.0" />
     <PackageVersion Include="Azure.Search.Documents" Version="11.6.0" />
-    <PackageVersion Include="Azure.Storage.Blobs" Version="12.21.2" />
-    <PackageVersion Include="Azure.Storage.Queues" Version="12.19.1" />
+    <PackageVersion Include="Azure.Storage.Blobs" Version="12.22.0" />
+    <PackageVersion Include="Azure.Storage.Queues" Version="12.20.0" />
+    <PackageVersion Include="ClosedXML" Version="0.104.0-rc1" />
     <PackageVersion Include="Elastic.Clients.Elasticsearch" Version="8.12.1" />
-    <PackageVersion Include="Discord.Net" Version="3.15.3" />
+    <PackageVersion Include="Discord.Net" Version="3.16.0" />
+    <PackageVersion Include="DocumentFormat.OpenXml" Version="3.1.0" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.11.65" />
     <PackageVersion Include="LLamaSharp" Version="0.16.0" />
     <PackageVersion Include="LLamaSharp.Backend.Cpu" Version="0.16.0" />
@@ -33,45 +35,31 @@
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
     <PackageVersion Include="Microsoft.Extensions.Logging.TraceSource" Version="8.0.0" />
-    <PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24271.1" />
-    <PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.72.240904.1" />
-    <PackageVersion Include="Microsoft.KernelMemory.Service.AspNetCore" Version="0.72.240904.1" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.73.240906.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.Service.AspNetCore" Version="0.73.240906.1" />
     <PackageVersion Include="MongoDB.Driver.GridFS" Version="2.28.0" />
     <PackageVersion Include="Moq" Version="4.20.72" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageVersion Include="OllamaSharp" Version="3.0.4" />
+    <PackageVersion Include="OllamaSharp" Version="3.0.7" />
     <PackageVersion Include="PdfPig" Version="0.1.8" />
     <PackageVersion Include="Pgvector" Version="0.3.0" />
     <PackageVersion Include="Polly.Core" Version="8.4.1" />
     <PackageVersion Include="RabbitMQ.Client" Version="6.8.1" />
-    <PackageVersion Include="NRedisStack" Version="0.12.0" />
+    <PackageVersion Include="NRedisStack" Version="0.13.0" />
     <PackageVersion Include="ReadLine" Version="2.0.1" />
     <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.7.3" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Memory.Data" Version="8.0.0" />
     <PackageVersion Include="System.Numerics.Tensors" Version="8.0.0" />
     <PackageVersion Include="System.Text.Json" Version="8.0.4" />
   </ItemGroup>
-  <!--
-    OpenXml version blocked, do not upgrade these
-    Excel import breaks with 'System.MissingMethodException: Method not found: 'DocumentFormat.OpenXml.OpenXmlElementList DocumentFormat.OpenXml.OpenXmlElement.get_ChildElements()
-    History:
-    - Sept 2024:
-      - ClosedXML upgraded from 0.102.2 to 0.102.3
-      - Tested OpenXml to 3.1.0 + ClosedXML 0.102.3 - Exception still occurring
-  -->
-  <ItemGroup>
-    <!-- see https://github.com/microsoft/kernel-memory/issues/265 -->
-    <PackageVersion Include="DocumentFormat.OpenXml" Version="2.20.0" />
-    <!-- see https://github.com/ClosedXML/ClosedXML/releases/tag/0.102.3 -->
-    <PackageVersion Include="ClosedXML" Version="0.102.3" />
-  </ItemGroup>
   <!-- Semantic Kernel -->
   <ItemGroup>
-    <PackageVersion Include="Microsoft.SemanticKernel" Version="1.18.2" />
-    <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.18.2" />
-    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.OpenAI" Version="1.18.2" />
-    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.AzureOpenAI" Version="1.18.2" />
+    <PackageVersion Include="Microsoft.SemanticKernel" Version="1.20.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.20.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.OpenAI" Version="1.20.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.AzureOpenAI" Version="1.20.0" />
   </ItemGroup>
   <!-- Documentation -->
   <ItemGroup>
@@ -89,11 +77,11 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
-    <PackageVersion Include="Roslynator.CodeAnalysis.Analyzers" Version="4.12.4">
+    <PackageVersion Include="Roslynator.CodeAnalysis.Analyzers" Version="4.12.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
-    <PackageVersion Include="Roslynator.Formatting.Analyzers" Version="4.12.4">
+    <PackageVersion Include="Roslynator.Formatting.Analyzers" Version="4.12.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
@@ -112,7 +100,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
-    <PackageVersion Include="Xunit.DependencyInjection" Version="9.3.1" />
+    <PackageVersion Include="Xunit.DependencyInjection" Version="9.4.0" />
     <PackageVersion Include="Xunit.DependencyInjection.Logging" Version="9.0.0" />
   </ItemGroup>
 </Project>
diff --git a/extensions/OpenAI/OpenAI/Tokenizers/DefaultGPTTokenizer.cs b/extensions/OpenAI/OpenAI/Tokenizers/DefaultGPTTokenizer.cs
@@ -9,7 +9,7 @@ namespace Microsoft.KernelMemory.AI.OpenAI;
 
 public static class DefaultGPTTokenizer
 {
-    private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel(
+    private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel(
         "gpt-4", new Dictionary<string, int> { { "<|im_start|>", 100264 }, { "<|im_end|>", 100265 } });
 
     public static int StaticCountTokens(string text)

diff --git a/extensions/OpenAI/OpenAI/Tokenizers/GPT2Tokenizer.cs b/extensions/OpenAI/OpenAI/Tokenizers/GPT2Tokenizer.cs
@@ -13,7 +13,7 @@ namespace Microsoft.KernelMemory.AI.OpenAI;
 /// </summary>
 public sealed class GPT2Tokenizer : ITextTokenizer
 {
-    private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("gpt2");
+    private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel("gpt2");
 
     /// <inheritdoc />
     public int CountTokens(string text)
@@ -24,6 +24,6 @@ public int CountTokens(string text)
     /// <inheritdoc />
     public IReadOnlyList<string> GetTokens(string text)
     {
-        return s_tokenizer.Encode(text, out string? _).Select(t => t.Value).ToList();
+        return s_tokenizer.EncodeToTokens(text, out string? _).Select(t => t.Value).ToList();
     }
 }
diff --git a/extensions/OpenAI/OpenAI/Tokenizers/GPT3Tokenizer.cs b/extensions/OpenAI/OpenAI/Tokenizers/GPT3Tokenizer.cs
@@ -13,7 +13,7 @@ namespace Microsoft.KernelMemory.AI.OpenAI;
 /// </summary>
 public sealed class GPT3Tokenizer : ITextTokenizer
 {
-    private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("text-davinci-003");
+    private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel("text-davinci-003");
 
     /// <inheritdoc />
     public int CountTokens(string text)
@@ -24,6 +24,6 @@ public int CountTokens(string text)
     /// <inheritdoc />
     public IReadOnlyList<string> GetTokens(string text)
     {
-        return s_tokenizer.Encode(text, out string? _).Select(t => t.Value).ToList();
+        return s_tokenizer.EncodeToTokens(text, out string? _).Select(t => t.Value).ToList();
     }
 }
diff --git a/extensions/OpenAI/OpenAI/Tokenizers/GPT4Tokenizer.cs b/extensions/OpenAI/OpenAI/Tokenizers/GPT4Tokenizer.cs
@@ -13,7 +13,7 @@ namespace Microsoft.KernelMemory.AI.OpenAI;
 /// </summary>
 public sealed class GPT4Tokenizer : ITextTokenizer
 {
-    private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4",
+    private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel("gpt-4",
         new Dictionary<string, int> { { "<|im_start|>", 100264 }, { "<|im_end|>", 100265 } });
 
     /// <inheritdoc />
@@ -25,6 +25,6 @@ public int CountTokens(string text)
     /// <inheritdoc />
     public IReadOnlyList<string> GetTokens(string text)
     {
-        return s_tokenizer.Encode(text, out string? _).Select(t => t.Value).ToList();
+        return s_tokenizer.EncodeToTokens(text, out string? _).Select(t => t.Value).ToList();
     }
 }
diff --git a/extensions/OpenAI/OpenAI/Tokenizers/GPT4oTokenizer.cs b/extensions/OpenAI/OpenAI/Tokenizers/GPT4oTokenizer.cs
@@ -14,7 +14,7 @@ namespace Microsoft.KernelMemory.AI.OpenAI;
 // ReSharper disable once InconsistentNaming
 public sealed class GPT4oTokenizer : ITextTokenizer
 {
-    private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4o",
+    private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel("gpt-4o",
         new Dictionary<string, int> { { "<|im_start|>", 100264 }, { "<|im_end|>", 100265 } });
 
     /// <inheritdoc />
@@ -26,6 +26,6 @@ public int CountTokens(string text)
     /// <inheritdoc />
     public IReadOnlyList<string> GetTokens(string text)
     {
-        return s_tokenizer.Encode(text, out string? _).Select(t => t.Value).ToList();
+        return s_tokenizer.EncodeToTokens(text, out string? _).Select(t => t.Value).ToList();
     }
 }
diff --git a/service/Core/Core.csproj b/service/Core/Core.csproj
@@ -5,7 +5,7 @@
         <RollForward>LatestMajor</RollForward>
         <AssemblyName>Microsoft.KernelMemory.Core</AssemblyName>
         <RootNamespace>Microsoft.KernelMemory</RootNamespace>
-        <NoWarn>$(NoWarn);KMEXP00;KMEXP01;KMEXP02;KMEXP03;KMEXP04;SKEXP0001;SKEXP0011;CA2208;CA1308;CA1724;</NoWarn>
+        <NoWarn>$(NoWarn);KMEXP00;KMEXP01;KMEXP02;KMEXP03;KMEXP04;SKEXP0001;SKEXP0011;CA2208;CA1308;CA1724;NU5104;</NoWarn>
     </PropertyGroup>
 
     <ItemGroup>