Fix heavy assets failing to download on slow connections

GDJS/Runtime/Model3DManager.ts

@@ -96,12 +96,6 @@ namespace gdjs {
       }
     }
 
-    /**
-     * Load all the 3D models.
-     *
-     * Note that even if a file is already loaded, it will be reloaded (useful for hot-reloading,
-     * as files can have been modified without the editor knowing).
-     */
     async loadResource(resourceName: string): Promise<void> {
       const resource = this._resourceLoader.getResource(resourceName);
       if (!resource) {
@@ -114,6 +108,9 @@ namespace gdjs {
       if (!loader) {
         return;
       }
+      if (this._loadedThreeModels.get(resource)) {
+        return;
+      }
       const url = this._resourceLoader.getFullUrl(resource.file);
       try {
         const response = await fetch(url, {

GDJS/Runtime/ResourceLoader.ts

@@ -27,6 +27,10 @@ namespace gdjs {
     );
   };
 
+  const maxForegroundConcurrency = 20;
+  const maxBackgroundConcurrency = 5;
+  const maxAttempt = 3;
+
   /**
    * A task of pre-loading resources used by a scene.
    *
@@ -129,6 +133,12 @@ namespace gdjs {
      * Only used by events.
      */
     private currentSceneLoadingProgress: float = 0;
+    /**
+     * It's set to `true` during intermediary loading screen to use a greater
+     * concurrency as the game is paused and doesn't need bandwidth (for video
+     * or music streaming or online multiplayer).
+     */
+    private _isLoadingInForeground = true;
 
     /**
      * @param runtimeGame The game.
@@ -218,13 +228,16 @@ namespace gdjs {
       onProgress: (loadingCount: integer, totalCount: integer) => void
     ): Promise<void> {
       let loadedCount = 0;
-      await Promise.all(
-        [...this._resources.values()].map(async (resource) => {
+      await processAndRetryIfNeededWithPromisePool(
+        [...this._resources.values()],
+        maxForegroundConcurrency,
+        maxAttempt,
+        async (resource) => {
           await this._loadResource(resource);
           await this._processResource(resource);
           loadedCount++;
           onProgress(loadedCount, this._resources.size);
-        })
+        }
       );
       this._sceneNamesToLoad.clear();
       this._sceneNamesToMakeReady.clear();
@@ -246,8 +259,11 @@ namespace gdjs {
       }
       let loadedCount = 0;
       const resources = [...this._globalResources, ...sceneResources.values()];
-      await Promise.all(
-        resources.map(async (resourceName) => {
+      await processAndRetryIfNeededWithPromisePool(
+        resources,
+        maxForegroundConcurrency,
+        maxAttempt,
+        async (resourceName) => {
           const resource = this._resources.get(resourceName);
           if (!resource) {
             logger.warn('Unable to find resource "' + resourceName + '".');
@@ -257,7 +273,7 @@ namespace gdjs {
           await this._processResource(resource);
           loadedCount++;
           onProgress(loadedCount, resources.length);
-        })
+        }
       );
       this._setSceneAssetsLoaded(firstSceneName);
       this._setSceneAssetsReady(firstSceneName);
@@ -307,8 +323,13 @@ namespace gdjs {
         return;
       }
       let loadedCount = 0;
-      await Promise.all(
-        [...sceneResources.values()].map(async (resourceName) => {
+      await processAndRetryIfNeededWithPromisePool(
+        [...sceneResources.values()],
+        this._isLoadingInForeground
+        ? maxForegroundConcurrency
+          : maxBackgroundConcurrency,
+        maxAttempt,
+        async (resourceName) => {
           const resource = this._resources.get(resourceName);
           if (!resource) {
             logger.warn('Unable to find resource "' + resourceName + '".');
@@ -318,7 +339,7 @@ namespace gdjs {
           loadedCount++;
           this.currentSceneLoadingProgress = loadedCount / this._resources.size;
           onProgress && (await onProgress(loadedCount, this._resources.size));
-        })
+        }
       );
       this._setSceneAssetsLoaded(sceneName);
     }
@@ -385,13 +406,16 @@ namespace gdjs {
       sceneName: string,
       onProgress?: (count: number, total: number) => void
     ): Promise<void> {
+      this._isLoadingInForeground = true;
       const task = this._prioritizeScene(sceneName);
       return new Promise<void>((resolve, reject) => {
         if (!task) {
+          this._isLoadingInForeground = false;
           resolve();
           return;
         }
         task.registerCallback(() => {
+          this._isLoadingInForeground = false;
           resolve();
         }, onProgress);
       });
@@ -553,4 +577,79 @@ namespace gdjs {
       return this._model3DManager;
     }
   }
+
+  type PromiseError<T> = { item: T; error: Error };
+
+  type PromisePoolOutput<T, U> = {
+    results: Array<U>;
+    errors: Array<PromiseError<T>>;
+  };
+
+  const processWithPromisePool = <T, U>(
+    items: Array<T>,
+    maxConcurrency: number,
+    asyncFunction: (item: T) => Promise<U>
+  ): Promise<PromisePoolOutput<T, U>> => {
+    const results: Array<U> = [];
+    const errors: Array<PromiseError<T>> = [];
+    let activePromises = 0;
+    let index = 0;
+
+    return new Promise((resolve, reject) => {
+      const executeNext = () => {
+        if (items.length === 0) {
+          resolve({ results, errors });
+          return;
+        }
+        while (activePromises < maxConcurrency && index < items.length) {
+          const item = items[index++];
+          activePromises++;
+
+          asyncFunction(item)
+            .then((result) => results.push(result))
+            .catch((error) => errors.push({ item, error }))
+            .finally(() => {
+              activePromises--;
+              if (index === items.length && activePromises === 0) {
+                resolve({ results, errors });
+              } else {
+                executeNext();
+              }
+            });
+        }
+      };
+
+      executeNext();
+    });
+  };
+
+  const processAndRetryIfNeededWithPromisePool = async <T, U>(
+    items: Array<T>,
+    maxConcurrency: number,
+    maxAttempt: number,
+    asyncFunction: (item: T) => Promise<U>
+  ): Promise<PromisePoolOutput<T, U>> => {
+    const output = await processWithPromisePool<T, U>(
+      items,
+      maxConcurrency,
+      asyncFunction
+    );
+    if (output.errors.length !== 0) {
+      logger.warn("Some assets couldn't be downloaded. Trying again now.");
+    }
+    for (
+      let attempt = 1;
+      attempt < maxAttempt && output.errors.length !== 0;
+      attempt++
+    ) {
+      const retryOutput = await processWithPromisePool<T, U>(
+        items,
+        maxConcurrency,
+        asyncFunction
+      );
+      output.results.push.apply(output.results, retryOutput.results);
+      output.errors = retryOutput.errors;
+    }
+    return output;
+  };
 }