refactor: Remove hard-coded shard count (#502)

Currently, the `shard`/`chunk` count is hard-coded to 4 so that we can
fetch the block header and shards in parallel. This PR removes the
hard-coded value by using the chunk count specified in the block header
to fetch the relevant shards.

block-server/handler.js

@@ -35,16 +35,12 @@ const normalizeBlockHeight = function(block_height) {
 }
 
 const fetchStreamerMessage = async function(block_height, options) {
-    const blockPromise = fetchBlockPromise(block_height, options);
-    // hardcoding 4 shards to test performance
-    const shardsPromises = await fetchShardsPromises(block_height, 4, options); // block.chunks.length)
+    const block = await fetchBlockPromise(block_height, options);
+    const shards = await Promise.all(fetchShardsPromises(block_height, block.chunks.length, options))
 
-    const results = await Promise.all([blockPromise, ...shardsPromises]);
-    const block = results.shift();
-    const shards = results;
     return {
-        block: block,
-        shards: shards,
+        block,
+        shards,
     };
 }
 
@@ -101,4 +97,4 @@ const renameUnderscoreFieldsToCamelCase = function(value) {
         return newValue;
     }
     return value;
-}
+}

runner/src/lake-client/lake-client.test.ts

@@ -14,7 +14,7 @@ describe('LakeClient', () => {
       .mockReturnValueOnce({ // block
         Body: {
           transformToString: () => JSON.stringify({
-            chunks: [0],
+            chunks: [0, 1, 2, 3],
             header: {
               height: blockHeight,
               hash: blockHash,
@@ -88,7 +88,7 @@ describe('LakeClient', () => {
       .mockReturnValueOnce({ // block
         Body: {
           transformToString: () => JSON.stringify({
-            chunks: [0],
+            chunks: [0, 1, 2, 3],
             header: {
               height: blockHeight,
               hash: blockHash,
@@ -130,7 +130,7 @@ describe('LakeClient', () => {
       .mockReturnValueOnce({ // block
         Body: {
           transformToString: () => JSON.stringify({
-            chunks: [0],
+            chunks: [0, 1, 2, 3],
             header: {
               height: blockHeight,
               hash: blockHash,

runner/src/lake-client/lake-client.ts

@@ -15,13 +15,13 @@ export default class LakeClient {
     return blockHeight.toString().padStart(12, '0');
   }
 
-  private async fetchShardsPromises (blockHeight: number, numberOfShards: number): Promise<Array<Promise<any>>> {
+  private fetchShards (blockHeight: number, numberOfShards: number): Array<Promise<any>> {
     return ([...Array(numberOfShards).keys()].map(async (shardId) =>
-      await this.fetchShardPromise(blockHeight, shardId)
+      await this.fetchShard(blockHeight, shardId)
     ));
   }
 
-  private async fetchShardPromise (blockHeight: number, shardId: number): Promise<any> {
+  private async fetchShard (blockHeight: number, shardId: number): Promise<any> {
     const params = {
       Bucket: `near-lake-data-${this.network}`,
       Key: `${this.normalizeBlockHeight(blockHeight)}/shard_${shardId}.json`,
@@ -76,12 +76,9 @@ export default class LakeClient {
       }
     }
 
-    const blockPromise = this.fetchBlockPromise(blockHeight);
-    const shardsPromises = await this.fetchShardsPromises(blockHeight, 4);
+    const block = await this.fetchBlockPromise(blockHeight);
+    const shards = await Promise.all(this.fetchShards(blockHeight, block.chunks.length));
 
-    const results = await Promise.all([blockPromise, ...shardsPromises]);
-    const block = results.shift();
-    const shards = results;
     return Block.fromStreamerMessage({
       block,
       shards,