diff --git a/tasks/sync-granule/GranuleFetcher.js b/tasks/sync-granule/GranuleFetcher.js index 155a9696c2a..8d2956f5efa 100644 --- a/tasks/sync-granule/GranuleFetcher.js +++ b/tasks/sync-granule/GranuleFetcher.js @@ -222,6 +222,23 @@ class GranuleFetcher { (f) => syncChecksumFiles || !this.isChecksumFile(f) ); + // For testing + const fileDownloadPageSize = 10; //This will be an ENV or on the Provider config + + if (fileDownloadPageSize && filesToDownload.length > fileDownloadPageSize) { + // Download the first fileDownloadPageSize worth of filesToDownload, + // continue paginating until all downloaded. + let startPos = 0; + + while (startPos < filesToDownload.length) { + await ingestFiles(filesToDownload.slice(startPos, fileDownloadPageSize - 1)); + startPos += fileDownloadPageSize - 1; + } + } else { + await ingestFiles(filesToDownload); + } + + // The below becomes ingestFiles function const downloadPromises = filesToDownload.map((file) => this._ingestFile({ file, diff --git a/tasks/sync-granule/index.js b/tasks/sync-granule/index.js index 101e3c4ea0c..7d46ed35a4c 100644 --- a/tasks/sync-granule/index.js +++ b/tasks/sync-granule/index.js @@ -51,6 +51,8 @@ async function download({ + `granuleID: ${granules[0].granuleId}, ` ); + // TODO globalConnectionLimit is evaluated in proceed + // which compares locks on the bucket to provider.globalConnectonLimit const proceed = await lock.proceed(bucket, provider, granules[0].granuleId); if (!proceed) {