From e8065e4f3edbfba8e549b500f4e4a40f6a231442 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Tue, 19 Nov 2024 17:11:14 +0100 Subject: [PATCH] universe: optimize syncer call This commit allows for a much larger (32x) page size than before. This should allow us to fetch the complete set of roots even faster, given that we should have 100% cache hits for the syncer calls now. We also reduce the overall count of requests by 1 in almost all cases by detecting that the returned page isn't full anymore (instead of reading until we get an empty page). --- universe/interface.go | 2 +- universe/syncer.go | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/universe/interface.go b/universe/interface.go index abeca1f48..8192078ec 100644 --- a/universe/interface.go +++ b/universe/interface.go @@ -38,7 +38,7 @@ var ( const ( // MaxPageSize is the maximum page size that can be used when querying // for asset roots and leaves. - MaxPageSize = 512 + MaxPageSize = 16384 ) // IdentifierKey is the compact representation of a universe identifier that can diff --git a/universe/syncer.go b/universe/syncer.go index 64a69cf90..104b24890 100644 --- a/universe/syncer.go +++ b/universe/syncer.go @@ -525,7 +525,9 @@ func (s *SimpleSyncer) SyncUniverse(ctx context.Context, host ServerAddr, // fetchAllRoots fetches all the roots from the remote Universe. This function // is used in order to isolate any logic related to the specifics of how we // fetch the data from the universe server. -func (s *SimpleSyncer) fetchAllRoots(ctx context.Context, diffEngine DiffEngine) ([]Root, error) { +func (s *SimpleSyncer) fetchAllRoots(ctx context.Context, + diffEngine DiffEngine) ([]Root, error) { + offset := int32(0) pageSize := defaultPageSize roots := make([]Root, 0) @@ -533,6 +535,7 @@ func (s *SimpleSyncer) fetchAllRoots(ctx context.Context, diffEngine DiffEngine) for { log.Debugf("Fetching roots in range: %v to %v", offset, offset+pageSize) + tempRoots, err := diffEngine.RootNodes( ctx, RootNodesQuery{ WithAmountsById: false, @@ -541,16 +544,17 @@ func (s *SimpleSyncer) fetchAllRoots(ctx context.Context, diffEngine DiffEngine) Limit: pageSize, }, ) - if err != nil { return nil, err } - if len(tempRoots) == 0 { + roots = append(roots, tempRoots...) + + // If we're getting a partial page, then we know we're done. + if len(tempRoots) < int(pageSize) { break } - roots = append(roots, tempRoots...) offset += pageSize }