-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Use a stack channel for deserialization * multi-threaded * add object dictionary pool * more pooling * adjust sqlite transport * format * Optimize IsPropNameValid * object loader first pass * save test * add cache pre check * save better deserialize * mostly works * uses tasks but slower at end * rework to make more sense * add check to avoid multi-deserialize * modify max parallelism * async enqueuing of tasks * switch to more asyncenumerable * fmt * fmt * cleanup sqlite * make ServerObjectManager * revert change * add ability to skip cache check * cache json to know what is loaded * testing * clean up usage * clean up and added new op * Fix exception handling * fixing progress * remove codejam * Hides ObjectPool dependency * fmt * Use the 1.0 BCL async to try to be more compatible * rename to dependencies * Move Polly to internal dependencies * format * remove more old references * remove stackchannel * fixes for registration * remove console writeline * add cache check shortcut for root object * recevie2 benchmark * add test for deserialize new * Use same asyncinterfaces as Dynamo. Merge fixes * clean up * fix download object progress * add channels * Use net48, netstandard2.1 and net8 * need AsyncInterfaces for net48 :( * options changes * revert to netstandard2.0 and net8.0 * fix totals * revert httpcontext changes * format * add active tasks collection * format * remove more changes and remove unused method * remove more comment changes --------- Co-authored-by: Jedd Morgan <[email protected]>
- Loading branch information
1 parent
e10cc38
commit f5a8d1a
Showing
26 changed files
with
227 additions
and
438 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
33 changes: 33 additions & 0 deletions
33
src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
using Open.ChannelExtensions; | ||
|
||
namespace Speckle.Sdk.Serialisation.V2.Receive; | ||
|
||
public abstract class ChannelLoader | ||
{ | ||
private const int HTTP_ID_CHUNK_SIZE = 500; | ||
private const int MAX_PARALLELISM_HTTP = 4; | ||
|
||
protected async Task GetAndCache(IEnumerable<string> allChildrenIds, CancellationToken cancellationToken = default) => | ||
await allChildrenIds | ||
.ToChannel(cancellationToken: cancellationToken) | ||
.Pipe(Environment.ProcessorCount, CheckCache, cancellationToken: cancellationToken) | ||
.Filter(x => x is not null) | ||
.Batch(HTTP_ID_CHUNK_SIZE) | ||
.WithTimeout(TimeSpan.FromSeconds(2)) | ||
.PipeAsync( | ||
MAX_PARALLELISM_HTTP, | ||
async x => await DownloadAndCache(x).ConfigureAwait(false), | ||
-1, | ||
false, | ||
cancellationToken | ||
) | ||
.Join() | ||
.ReadAllConcurrently(Environment.ProcessorCount, SaveToCache, cancellationToken) | ||
.ConfigureAwait(false); | ||
|
||
public abstract string? CheckCache(string id); | ||
|
||
public abstract Task<List<(string, string)>> DownloadAndCache(List<string?> ids); | ||
|
||
public abstract void SaveToCache((string, string) x); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.