diff --git a/Directory.Packages.props b/Directory.Packages.props
index 80bd0db1..97872873 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -13,13 +13,13 @@
-
+
-
+
@@ -28,9 +28,10 @@
+
-
+
\ No newline at end of file
diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json
index a8bfa0e1..bf3c01fd 100644
--- a/src/Speckle.Objects/packages.lock.json
+++ b/src/Speckle.Objects/packages.lock.json
@@ -484,7 +484,6 @@
"type": "Project",
"dependencies": {
"GraphQL.Client": "[6.0.0, )",
- "Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )",
"Microsoft.CSharp": "[4.7.0, )",
"Microsoft.Data.Sqlite": "[7.0.7, )",
"Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )",
@@ -508,12 +507,6 @@
"System.Reactive": "5.0.0"
}
},
- "Microsoft.Bcl.AsyncInterfaces": {
- "type": "CentralTransitive",
- "requested": "[5.0.0, )",
- "resolved": "5.0.0",
- "contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ=="
- },
"Microsoft.CSharp": {
"type": "CentralTransitive",
"requested": "[4.7.0, )",
diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs
new file mode 100644
index 00000000..d7bc96df
--- /dev/null
+++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs
@@ -0,0 +1,33 @@
+using Open.ChannelExtensions;
+
+namespace Speckle.Sdk.Serialisation.V2.Receive;
+
+public abstract class ChannelLoader
+{
+ private const int HTTP_ID_CHUNK_SIZE = 500;
+ private const int MAX_PARALLELISM_HTTP = 4;
+
+ protected async Task GetAndCache(IEnumerable allChildrenIds, CancellationToken cancellationToken = default) =>
+ await allChildrenIds
+ .ToChannel(cancellationToken: cancellationToken)
+ .Pipe(Environment.ProcessorCount, CheckCache, cancellationToken: cancellationToken)
+ .Filter(x => x is not null)
+ .Batch(HTTP_ID_CHUNK_SIZE)
+ .WithTimeout(TimeSpan.FromSeconds(2))
+ .PipeAsync(
+ MAX_PARALLELISM_HTTP,
+ async x => await DownloadAndCache(x).ConfigureAwait(false),
+ -1,
+ false,
+ cancellationToken
+ )
+ .Join()
+ .ReadAllConcurrently(Environment.ProcessorCount, SaveToCache, cancellationToken)
+ .ConfigureAwait(false);
+
+ public abstract string? CheckCache(string id);
+
+ public abstract Task> DownloadAndCache(List ids);
+
+ public abstract void SaveToCache((string, string) x);
+}
diff --git a/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj b/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj
index 3b0d9060..9299b933 100644
--- a/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj
+++ b/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj
@@ -1,4 +1,4 @@
-
+
netstandard2.0;net8.0
@@ -27,7 +27,9 @@
-
+
+
+
diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json
index 0c481e76..80f29e97 100644
--- a/src/Speckle.Sdk.Dependencies/packages.lock.json
+++ b/src/Speckle.Sdk.Dependencies/packages.lock.json
@@ -42,6 +42,17 @@
"Microsoft.NETCore.Platforms": "1.1.0"
}
},
+ "Open.ChannelExtensions": {
+ "type": "Direct",
+ "requested": "[8.5.0, )",
+ "resolved": "8.5.0",
+ "contentHash": "dKD2iNfUYw+aOvwM2vCnD+q6JCtHiabkufKM1GateedRzcgv0RrtA4MoJI+7Y8N21R5A+wUA+j6P88g6mXPavA==",
+ "dependencies": {
+ "Microsoft.Bcl.AsyncInterfaces": "8.0.0",
+ "System.Collections.Immutable": "8.0.0",
+ "System.Threading.Channels": "8.0.0"
+ }
+ },
"Polly": {
"type": "Direct",
"requested": "[7.2.3, )",
@@ -75,6 +86,15 @@
"resolved": "0.9.6",
"contentHash": "HKH7tYrYYlCK1ct483hgxERAdVdMtl7gUKW9ijWXxA1UsYR4Z+TrRHYmzZ9qmpu1NnTycSrp005NYM78GDKV1w=="
},
+ "System.Threading.Channels": {
+ "type": "Direct",
+ "requested": "[8.0.0, )",
+ "resolved": "8.0.0",
+ "contentHash": "CMaFr7v+57RW7uZfZkPExsPB6ljwzhjACWW1gfU35Y56rk72B/Wu+sTqxVmGSk4SFUlPc3cjeKND0zktziyjBA==",
+ "dependencies": {
+ "System.Threading.Tasks.Extensions": "4.5.4"
+ }
+ },
"ILRepack": {
"type": "Transitive",
"resolved": "2.0.33",
@@ -94,6 +114,57 @@
"type": "Transitive",
"resolved": "8.0.0",
"contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw=="
+ },
+ "System.Buffers": {
+ "type": "Transitive",
+ "resolved": "4.5.1",
+ "contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg=="
+ },
+ "System.Collections.Immutable": {
+ "type": "Transitive",
+ "resolved": "8.0.0",
+ "contentHash": "AurL6Y5BA1WotzlEvVaIDpqzpIPvYnnldxru8oXJU2yFxFUy3+pNXjXd1ymO+RA0rq0+590Q8gaz2l3Sr7fmqg==",
+ "dependencies": {
+ "System.Memory": "4.5.5",
+ "System.Runtime.CompilerServices.Unsafe": "6.0.0"
+ }
+ },
+ "System.Memory": {
+ "type": "Transitive",
+ "resolved": "4.5.5",
+ "contentHash": "XIWiDvKPXaTveaB7HVganDlOCRoj03l+jrwNvcge/t8vhGYKvqV+dMv6G4SAX2NoNmN0wZfVPTAlFwZcZvVOUw==",
+ "dependencies": {
+ "System.Buffers": "4.5.1",
+ "System.Numerics.Vectors": "4.4.0",
+ "System.Runtime.CompilerServices.Unsafe": "4.5.3"
+ }
+ },
+ "System.Numerics.Vectors": {
+ "type": "Transitive",
+ "resolved": "4.4.0",
+ "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ=="
+ },
+ "System.Runtime.CompilerServices.Unsafe": {
+ "type": "Transitive",
+ "resolved": "6.0.0",
+ "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg=="
+ },
+ "System.Threading.Tasks.Extensions": {
+ "type": "Transitive",
+ "resolved": "4.5.4",
+ "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==",
+ "dependencies": {
+ "System.Runtime.CompilerServices.Unsafe": "4.5.3"
+ }
+ },
+ "Microsoft.Bcl.AsyncInterfaces": {
+ "type": "CentralTransitive",
+ "requested": "[5.0.0, )",
+ "resolved": "8.0.0",
+ "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==",
+ "dependencies": {
+ "System.Threading.Tasks.Extensions": "4.5.4"
+ }
}
},
"net8.0": {
@@ -128,6 +199,12 @@
"Microsoft.SourceLink.Common": "8.0.0"
}
},
+ "Open.ChannelExtensions": {
+ "type": "Direct",
+ "requested": "[8.5.0, )",
+ "resolved": "8.5.0",
+ "contentHash": "dKD2iNfUYw+aOvwM2vCnD+q6JCtHiabkufKM1GateedRzcgv0RrtA4MoJI+7Y8N21R5A+wUA+j6P88g6mXPavA=="
+ },
"Polly": {
"type": "Direct",
"requested": "[7.2.3, )",
@@ -161,6 +238,12 @@
"resolved": "0.9.6",
"contentHash": "HKH7tYrYYlCK1ct483hgxERAdVdMtl7gUKW9ijWXxA1UsYR4Z+TrRHYmzZ9qmpu1NnTycSrp005NYM78GDKV1w=="
},
+ "System.Threading.Channels": {
+ "type": "Direct",
+ "requested": "[8.0.0, )",
+ "resolved": "8.0.0",
+ "contentHash": "CMaFr7v+57RW7uZfZkPExsPB6ljwzhjACWW1gfU35Y56rk72B/Wu+sTqxVmGSk4SFUlPc3cjeKND0zktziyjBA=="
+ },
"ILRepack": {
"type": "Transitive",
"resolved": "2.0.33",
diff --git a/src/Speckle.Sdk/Api/GraphQL/Client.cs b/src/Speckle.Sdk/Api/GraphQL/Client.cs
index 37fa525b..2dadff62 100644
--- a/src/Speckle.Sdk/Api/GraphQL/Client.cs
+++ b/src/Speckle.Sdk/Api/GraphQL/Client.cs
@@ -1,5 +1,4 @@
using System.Diagnostics.CodeAnalysis;
-using System.Dynamic;
using System.Net.WebSockets;
using System.Reflection;
using GraphQL;
@@ -175,46 +174,11 @@ internal void MaybeThrowFromGraphQLErrors(GraphQLRequest request, GraphQLResp
}
}
- private Dictionary ConvertExpandoToDict(ExpandoObject expando)
- {
- var variables = new Dictionary();
- foreach (KeyValuePair kvp in expando)
- {
- object? value;
- if (kvp.Value is ExpandoObject ex)
- {
- value = ConvertExpandoToDict(ex);
- }
- else
- {
- value = kvp.Value;
- }
-
- variables[kvp.Key] = value;
- }
- return variables;
- }
-
- /* private ILogEventEnricher[] CreateEnrichers(GraphQLRequest request)
- {
- // i know this is double (de)serializing, but we need a recursive convert to
- // dict here
- var expando = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(request.Variables));
- var variables = request.Variables != null && expando != null ? ConvertExpandoToDict(expando) : null;
- return new ILogEventEnricher[]
- {
- new PropertyEnricher("serverUrl", ServerUrl),
- new PropertyEnricher("graphqlQuery", request.Query),
- new PropertyEnricher("graphqlVariables", variables),
- new PropertyEnricher("resultType", typeof(T).Name)
- };
- }*/
-
IDisposable ISpeckleGraphQLClient.SubscribeTo(GraphQLRequest request, Action