Add cache download channels (#149)

* Use a stack channel for deserialization * multi-threaded * add object dictionary pool * more pooling * adjust sqlite transport * format * Optimize IsPropNameValid * object loader first pass * save test * add cache pre check * save better deserialize * mostly works * uses tasks but slower at end * rework to make more sense * add check to avoid multi-deserialize * modify max parallelism * async enqueuing of tasks * switch to more asyncenumerable * fmt * fmt * cleanup sqlite * make ServerObjectManager * revert change * add ability to skip cache check * cache json to know what is loaded * testing * clean up usage * clean up and added new op * Fix exception handling * fixing progress * remove codejam * Hides ObjectPool dependency * fmt * Use the 1.0 BCL async to try to be more compatible * rename to dependencies * Move Polly to internal dependencies * format * remove more old references * remove stackchannel * fixes for registration * remove console writeline * add cache check shortcut for root object * recevie2 benchmark * add test for deserialize new * Use same asyncinterfaces as Dynamo. Merge fixes * clean up * fix download object progress * add channels * Use net48, netstandard2.1 and net8 * need AsyncInterfaces for net48 :( * options changes * revert to netstandard2.0 and net8.0 * fix totals * revert httpcontext changes * format * add active tasks collection * format * remove more changes and remove unused method * remove more comment changes --------- Co-authored-by: Jedd Morgan <[email protected]>
specklesystems · Oct 25, 2024 · f5a8d1a · f5a8d1a
1 parent e10cc38
commit f5a8d1a
Show file tree

Hide file tree

Showing 26 changed files with 227 additions and 438 deletions.
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -13,13 +13,13 @@
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="2.2.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.2.0" />
-    <PackageVersion Include="Microsoft.Extensions.Logging" Version="2.2.0" />
+    <PackageVersion Include="Microsoft.Extensions.Logging" Version="2.2.0" /> 
     <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
-    <PackageVersion Include="MongoDB.Driver" Version="2.19.2" />
     <PackageVersion Include="Moq" Version="4.20.70" />
     <PackageVersion Include="NUnit3TestAdapter" Version="4.6.0" />
     <PackageVersion Include="NUnit" Version="4.2.2" />
     <PackageVersion Include="NUnit.Analyzers" Version="4.2.0" />
+    <PackageVersion Include="Open.ChannelExtensions" Version="8.5.0" />
     <PackageVersion Include="Polly" Version="7.2.3" />
     <PackageVersion Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
     <PackageVersion Include="Polly.Extensions.Http" Version="3.0.0" />
@@ -28,9 +28,10 @@
     <PackageVersion Include="Speckle.DoubleNumerics" Version="4.0.1" />
     <PackageVersion Include="SimpleExec" Version="12.0.0" />
     <PackageVersion Include="Speckle.Objects" Version="3.1.0-dev.109" />
+    <PackageVersion Include="System.Threading.Channels" Version="8.0.0" />
     <GlobalPackageReference Include="PolySharp" Version="1.14.1" />
     <GlobalPackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
     <GlobalPackageReference Include="GitVersion.MsBuild" Version="5.12.0" />
     <GlobalPackageReference Include="Speckle.InterfaceGenerator" Version="0.9.6" />
   </ItemGroup>
-</Project>
+</Project>
diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json
@@ -484,7 +484,6 @@
         "type": "Project",
         "dependencies": {
           "GraphQL.Client": "[6.0.0, )",
-          "Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )",
           "Microsoft.CSharp": "[4.7.0, )",
           "Microsoft.Data.Sqlite": "[7.0.7, )",
           "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )",
@@ -508,12 +507,6 @@
           "System.Reactive": "5.0.0"
         }
       },
-      "Microsoft.Bcl.AsyncInterfaces": {
-        "type": "CentralTransitive",
-        "requested": "[5.0.0, )",
-        "resolved": "5.0.0",
-        "contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ=="
-      },
       "Microsoft.CSharp": {
         "type": "CentralTransitive",
         "requested": "[4.7.0, )",

diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs
@@ -0,0 +1,33 @@
+using Open.ChannelExtensions;
+
+namespace Speckle.Sdk.Serialisation.V2.Receive;
+
+public abstract class ChannelLoader
+{
+  private const int HTTP_ID_CHUNK_SIZE = 500;
+  private const int MAX_PARALLELISM_HTTP = 4;
+
+  protected async Task GetAndCache(IEnumerable<string> allChildrenIds, CancellationToken cancellationToken = default) =>
+    await allChildrenIds
+      .ToChannel(cancellationToken: cancellationToken)
+      .Pipe(Environment.ProcessorCount, CheckCache, cancellationToken: cancellationToken)
+      .Filter(x => x is not null)
+      .Batch(HTTP_ID_CHUNK_SIZE)
+      .WithTimeout(TimeSpan.FromSeconds(2))
+      .PipeAsync(
+        MAX_PARALLELISM_HTTP,
+        async x => await DownloadAndCache(x).ConfigureAwait(false),
+        -1,
+        false,
+        cancellationToken
+      )
+      .Join()
+      .ReadAllConcurrently(Environment.ProcessorCount, SaveToCache, cancellationToken)
+      .ConfigureAwait(false);
+
+  public abstract string? CheckCache(string id);
+
+  public abstract Task<List<(string, string)>> DownloadAndCache(List<string?> ids);
+
+  public abstract void SaveToCache((string, string) x);
+}
diff --git a/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj b/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj
@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup Label="Compiler Properties">
     <TargetFrameworks>netstandard2.0;net8.0</TargetFrameworks>
@@ -27,7 +27,9 @@
     <PackageReference Include="Microsoft.Extensions.ObjectPool" PrivateAssets="all" />    
     <PackageReference Include="Polly" PrivateAssets="all" />
     <PackageReference Include="Polly.Contrib.WaitAndRetry" PrivateAssets="all" />
-    <PackageReference Include="Polly.Extensions.Http" PrivateAssets="all"  />
+    <PackageReference Include="Polly.Extensions.Http" PrivateAssets="all" />
+    <PackageReference Include="Open.ChannelExtensions" PrivateAssets="all" />
+    <PackageReference Include="System.Threading.Channels" PrivateAssets="all" />
 
   </ItemGroup>
 </Project>
diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json
@@ -42,6 +42,17 @@
           "Microsoft.NETCore.Platforms": "1.1.0"
         }
       },
+      "Open.ChannelExtensions": {
+        "type": "Direct",
+        "requested": "[8.5.0, )",
+        "resolved": "8.5.0",
+        "contentHash": "dKD2iNfUYw+aOvwM2vCnD+q6JCtHiabkufKM1GateedRzcgv0RrtA4MoJI+7Y8N21R5A+wUA+j6P88g6mXPavA==",
+        "dependencies": {
+          "Microsoft.Bcl.AsyncInterfaces": "8.0.0",
+          "System.Collections.Immutable": "8.0.0",
+          "System.Threading.Channels": "8.0.0"
+        }
+      },
       "Polly": {
         "type": "Direct",
         "requested": "[7.2.3, )",
@@ -75,6 +86,15 @@
         "resolved": "0.9.6",
         "contentHash": "HKH7tYrYYlCK1ct483hgxERAdVdMtl7gUKW9ijWXxA1UsYR4Z+TrRHYmzZ9qmpu1NnTycSrp005NYM78GDKV1w=="
       },
+      "System.Threading.Channels": {
+        "type": "Direct",
+        "requested": "[8.0.0, )",
+        "resolved": "8.0.0",
+        "contentHash": "CMaFr7v+57RW7uZfZkPExsPB6ljwzhjACWW1gfU35Y56rk72B/Wu+sTqxVmGSk4SFUlPc3cjeKND0zktziyjBA==",
+        "dependencies": {
+          "System.Threading.Tasks.Extensions": "4.5.4"
+        }
+      },
       "ILRepack": {
         "type": "Transitive",
         "resolved": "2.0.33",
@@ -94,6 +114,57 @@
         "type": "Transitive",
         "resolved": "8.0.0",
         "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw=="
+      },
+      "System.Buffers": {
+        "type": "Transitive",
+        "resolved": "4.5.1",
+        "contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg=="
+      },
+      "System.Collections.Immutable": {
+        "type": "Transitive",
+        "resolved": "8.0.0",
+        "contentHash": "AurL6Y5BA1WotzlEvVaIDpqzpIPvYnnldxru8oXJU2yFxFUy3+pNXjXd1ymO+RA0rq0+590Q8gaz2l3Sr7fmqg==",
+        "dependencies": {
+          "System.Memory": "4.5.5",
+          "System.Runtime.CompilerServices.Unsafe": "6.0.0"
+        }
+      },
+      "System.Memory": {
+        "type": "Transitive",
+        "resolved": "4.5.5",
+        "contentHash": "XIWiDvKPXaTveaB7HVganDlOCRoj03l+jrwNvcge/t8vhGYKvqV+dMv6G4SAX2NoNmN0wZfVPTAlFwZcZvVOUw==",
+        "dependencies": {
+          "System.Buffers": "4.5.1",
+          "System.Numerics.Vectors": "4.4.0",
+          "System.Runtime.CompilerServices.Unsafe": "4.5.3"
+        }
+      },
+      "System.Numerics.Vectors": {
+        "type": "Transitive",
+        "resolved": "4.4.0",
+        "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ=="
+      },
+      "System.Runtime.CompilerServices.Unsafe": {
+        "type": "Transitive",
+        "resolved": "6.0.0",
+        "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg=="
+      },
+      "System.Threading.Tasks.Extensions": {
+        "type": "Transitive",
+        "resolved": "4.5.4",
+        "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==",
+        "dependencies": {
+          "System.Runtime.CompilerServices.Unsafe": "4.5.3"
+        }
+      },
+      "Microsoft.Bcl.AsyncInterfaces": {
+        "type": "CentralTransitive",
+        "requested": "[5.0.0, )",
+        "resolved": "8.0.0",
+        "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==",
+        "dependencies": {
+          "System.Threading.Tasks.Extensions": "4.5.4"
+        }
       }
     },
     "net8.0": {
@@ -128,6 +199,12 @@
           "Microsoft.SourceLink.Common": "8.0.0"
         }
       },
+      "Open.ChannelExtensions": {
+        "type": "Direct",
+        "requested": "[8.5.0, )",
+        "resolved": "8.5.0",
+        "contentHash": "dKD2iNfUYw+aOvwM2vCnD+q6JCtHiabkufKM1GateedRzcgv0RrtA4MoJI+7Y8N21R5A+wUA+j6P88g6mXPavA=="
+      },
       "Polly": {
         "type": "Direct",
         "requested": "[7.2.3, )",
@@ -161,6 +238,12 @@
         "resolved": "0.9.6",
         "contentHash": "HKH7tYrYYlCK1ct483hgxERAdVdMtl7gUKW9ijWXxA1UsYR4Z+TrRHYmzZ9qmpu1NnTycSrp005NYM78GDKV1w=="
       },
+      "System.Threading.Channels": {
+        "type": "Direct",
+        "requested": "[8.0.0, )",
+        "resolved": "8.0.0",
+        "contentHash": "CMaFr7v+57RW7uZfZkPExsPB6ljwzhjACWW1gfU35Y56rk72B/Wu+sTqxVmGSk4SFUlPc3cjeKND0zktziyjBA=="
+      },
       "ILRepack": {
         "type": "Transitive",
         "resolved": "2.0.33",

diff --git a/src/Speckle.Sdk/Api/GraphQL/Client.cs b/src/Speckle.Sdk/Api/GraphQL/Client.cs
@@ -1,5 +1,4 @@
 using System.Diagnostics.CodeAnalysis;
-using System.Dynamic;
 using System.Net.WebSockets;
 using System.Reflection;
 using GraphQL;
@@ -175,46 +174,11 @@ internal void MaybeThrowFromGraphQLErrors<T>(GraphQLRequest request, GraphQLResp
     }
   }
 
-  private Dictionary<string, object?> ConvertExpandoToDict(ExpandoObject expando)
-  {
-    var variables = new Dictionary<string, object?>();
-    foreach (KeyValuePair<string, object?> kvp in expando)
-    {
-      object? value;
-      if (kvp.Value is ExpandoObject ex)
-      {
-        value = ConvertExpandoToDict(ex);
-      }
-      else
-      {
-        value = kvp.Value;
-      }
-
-      variables[kvp.Key] = value;
-    }
-    return variables;
-  }
-
-  /* private ILogEventEnricher[] CreateEnrichers<T>(GraphQLRequest request)
-   {
-     // i know this is double  (de)serializing, but we need a recursive convert to
-     // dict<str, object> here
-     var expando = JsonConvert.DeserializeObject<ExpandoObject>(JsonConvert.SerializeObject(request.Variables));
-     var variables = request.Variables != null && expando != null ? ConvertExpandoToDict(expando) : null;
-     return new ILogEventEnricher[]
-     {
-       new PropertyEnricher("serverUrl", ServerUrl),
-       new PropertyEnricher("graphqlQuery", request.Query),
-       new PropertyEnricher("graphqlVariables", variables),
-       new PropertyEnricher("resultType", typeof(T).Name)
-     };
-   }*/
-
   IDisposable ISpeckleGraphQLClient.SubscribeTo<T>(GraphQLRequest request, Action<object, T> callback) =>
     SubscribeTo(request, callback);
 
   /// <inheritdoc cref="ISpeckleGraphQLClient.SubscribeTo{T}"/>
-  internal IDisposable SubscribeTo<T>(GraphQLRequest request, Action<object, T> callback)
+  private IDisposable SubscribeTo<T>(GraphQLRequest request, Action<object, T> callback)
   {
     //using (LogContext.Push(CreateEnrichers<T>(request)))
     {

diff --git a/src/Speckle.Sdk/Common/HashCode.cs b/src/Speckle.Sdk/Common/HashCode.cs
@@ -63,7 +63,7 @@ public static HashCode OfEach<T>(IEnumerable<T>? items) =>
   /// <typeparam name="T">The type of the item.</typeparam>
   /// <param name="item">The item.</param>
   /// <returns>The new hash code.</returns>
-  public HashCode And<T>(T item) => new HashCode(CombineHashCodes(this._value, GetHashCode(item)));
+  public HashCode And<T>(T item) => new HashCode(CombineHashCodes(_value, GetHashCode(item)));
 
   /// <summary>
   /// Adds the hash code of the specified items in the collection.
@@ -75,21 +75,21 @@ public HashCode AndEach<T>(IEnumerable<T>? items)
   {
     if (items == null)
     {
-      return new HashCode(this._value);
+      return new HashCode(_value);
     }
 
-    return new HashCode(GetHashCode(items, this._value));
+    return new HashCode(GetHashCode(items, _value));
   }
 
   /// <inheritdoc />
-  public bool Equals(HashCode other) => this._value.Equals(other._value);
+  public bool Equals(HashCode other) => _value.Equals(other._value);
 
   /// <inheritdoc />
   public override bool Equals(object? obj)
   {
     if (obj is HashCode code)
     {
-      return this.Equals(code);
+      return Equals(code);
     }
 
     return false;

diff --git a/src/Speckle.Sdk/Credentials/Account.cs b/src/Speckle.Sdk/Credentials/Account.cs
@@ -89,7 +89,7 @@ public override bool Equals(object obj)
 
   public override int GetHashCode()
   {
-#if NETSTANDARD2_0
+#if  NETSTANDARD2_0
     return Speckle.Sdk.Common.HashCode.Of(userInfo.email).And(serverInfo.url);
 #else
     return HashCode.Combine(userInfo.email, serverInfo.url);

diff --git a/src/Speckle.Sdk/Credentials/AccountManager.cs b/src/Speckle.Sdk/Credentials/AccountManager.cs
@@ -835,12 +835,12 @@ private async Task<bool> IsFrontend2Server(Uri server)
 
   private static string GenerateChallenge()
   {
-#if NETSTANDARD2_0
+#if NET8_0
+    byte[] challengeData = RandomNumberGenerator.GetBytes(32);
+#else
     using RNGCryptoServiceProvider rng = new();
     byte[] challengeData = new byte[32];
     rng.GetBytes(challengeData);
-#else
-    byte[] challengeData = RandomNumberGenerator.GetBytes(32);
 #endif
     //escaped chars like % do not play nice with the server
     return Regex.Replace(Convert.ToBase64String(challengeData), @"[^\w\.@-]", "");

diff --git a/src/Speckle.Sdk/Helpers/PropNameValidator.cs b/src/Speckle.Sdk/Helpers/PropNameValidator.cs
@@ -35,9 +35,9 @@ public static bool IsChunkable(string propName, out int chunkSize)
 
   [Pure]
   public static bool IsDetached(string propName) =>
-#if NET5_0_OR_GREATER
-    propName.StartsWith('@');
-#else
+#if NETSTANDARD2_0
     propName.StartsWith("@");
+#else
+    propName.StartsWith('@');
 #endif
 }
diff --git a/src/Speckle.Sdk/Models/GraphTraversal/RuleBuilder.cs b/src/Speckle.Sdk/Models/GraphTraversal/RuleBuilder.cs
@@ -20,7 +20,7 @@ private TraversalRule()
 
   public ITraversalBuilderReturn ContinueTraversing(SelectMembers membersToTraverse)
   {
-    this._membersToTraverse = membersToTraverse;
+    _membersToTraverse = membersToTraverse;
     return this;
   }