bug: In safari the canvas uses too much memory #61

.github/workflows/on_release_build_image.yml

@@ -6,7 +6,7 @@ name: Create and publish a Docker image
 
 on:
   push:
-    branches: ["main"]
+    branches: ["main", "61-bug-in-safari-the-canvas-uses-too-much-memory"]
   release:
     types: [published]
 

src/app/enums/mime-types.enum.ts

@@ -4,5 +4,5 @@ export enum MimeTypes {
     audioWebm = 'audio/webm;codecs=opus',
     audioMp4 = 'audio/mp4;codecs=mp4a.67',
     audioMp3 = 'audio/mpeg',
-    imagePng = 'image/png'
+    imageJpeg = 'image/jpeg'
 }

src/app/models/animator.ts

@@ -40,6 +40,8 @@ export class Animator {
     videoStream: MediaStream;
     width: any;
     zeroPlayTime: number;
+    imageCanvas: any;
+    context: any;
 
     private isAnimatorPlaying: BehaviorSubject<boolean>;
     private frameRate: BehaviorSubject<number>;
@@ -135,36 +137,55 @@ export class Animator {
             this.videoStream = stream;
             this.isStreaming = true;
 
+            this.setupContext()
+
             return stream;
         } catch (err) {
             console.error(err);
             throw err;
         }
     }
 
+    private setupContext() {
+        this.imageCanvas = document.createElement('canvas');
+        this.imageCanvas.id = "capture-from-video-canvas"
+        this.imageCanvas.width = this.width;
+        this.imageCanvas.height = this.height;
+
+        this.context = this.imageCanvas.getContext('2d', { alpha: false });
+        if (this.rotated) {
+            this.context.rotate(Math.PI);
+            this.context.translate(-this.width, -this.height);
+        }
+    }
+
     /*
     * Method is used to capture new image and create canvas out of it and share it with other components
     */
     public async capture() {
         // console.log('🚀 ~ file: animator.ts ~ line 103 ~ Animator ~ capture ~ capture');
         if (!this.isStreaming) { return; }
-        const imageCanvas: HTMLCanvasElement = document.createElement('canvas');
-        imageCanvas.width = this.width;
-        imageCanvas.height = this.height;
-        const context = imageCanvas.getContext('2d', { alpha: false });
-        if (this.rotated) {
-            context.rotate(Math.PI);
-            context.translate(-this.width, -this.height);
-        }
-        context.drawImage(this.video, 0, 0, this.width, this.height);
-        this.frames.push(imageCanvas);
+
+        this.context.drawImage(this.video, 0, 0, this.width, this.height);
+
+        this.imageCanvas.toBlob((blob: Blob) => { 
+              var img = new Image();
+
+              const dataUrl = URL.createObjectURL(blob);
+
+              img.onload = async() => {
+                this.frames.push(img);
+                URL.revokeObjectURL(dataUrl);
+              }
+              img.src = dataUrl;
+
+              this.snapshotContext.clearRect(0, 0, this.width, this.height);
+              this.snapshotContext.drawImage(this.imageCanvas, 0, 0, this.width, this.height);
+
+              this.frameWebps.push(blob);
+
+        }, 'image/jpeg', 0.8);
 
-        const promise = await new Promise(((resolve, reject) => {
-            this.snapshotContext.clearRect(0, 0, this.width, this.height);
-            this.snapshotContext.drawImage(imageCanvas, 0, 0, this.width, this.height);
-            imageCanvas.toBlob(blob => { resolve(blob); }, 'image/webp');
-        }));
-        this.frameWebps.push(promise);
         return this.frames;
     }
 
@@ -448,16 +469,18 @@ export class Animator {
             // If not specified this defaults to the same value as `quality`.
         });
 
-        if (this.isSafari()) {
-            const convertedFrames = await this.videoService.convertPngToWebP(this.frameWebps);
+        // ::TODO:: in the past, we only converted the frames for Safari. Right now, we always create jpegs.
+        // This simplifies the process quite a bit, as it's more consistent. However, we should check if this causes further issues.
+        //if (this.isSafari()) {
+            const convertedFrames = await this.videoService.convertPotentiallyMixedFrames(this.frameWebps);
             for (const frame of convertedFrames) {
                 videoWriter.addFrame(this.uint8ToBase64(frame));
             }
-        } else {
+        /*} else {
             for (const frame of this.frames) {
                 videoWriter.addFrame(frame);
             }
-        }
+        }*/
 
         const blob = await videoWriter.complete();
         return blob;
@@ -551,7 +574,7 @@ export class Animator {
     */
     private addFrameVP8(frameOffset: number, callback: any, blob: Blob, index: number) {
         let blobURL = URL.createObjectURL(blob);
-        const image = new Image(this.width, this.height);
+        const image = new Image();
         this.framesInFlight++;
         image.addEventListener('error', (error) => {
             if (image.getAttribute('triedvp8l')) {
@@ -571,11 +594,8 @@ export class Animator {
         });
 
         image.addEventListener('load', async (evt: any) => {
-            const newCanvas = document.createElement('canvas');
-            newCanvas.width = this.width;
-            newCanvas.height = this.height;
-            newCanvas.getContext('2d', { alpha: false }).drawImage(evt.target, 0, 0, this.width, this.height);
-            this.frames[frameOffset + index] = newCanvas;
+            this.frames[frameOffset + index] = image
+
             this.frameWebps[frameOffset + index] = await new Promise((resolve, reject) => {
                 resolve(blob);
             });

src/app/pages/animator/components/thumbnail/thumbnail.component.html

@@ -1,4 +1,4 @@
 <div (click)="onClick()">
-  <canvas #thumbnail width="width" height="height"></canvas>
+  <div #thumbnail width="width" height="height"></div>
   <ion-icon src="../../../assets/icons/custom/delete.svg" color="light"></ion-icon>
-</div>
+</div>

src/app/pages/animator/components/thumbnail/thumbnail.component.ts

@@ -9,11 +9,10 @@ import { AnimatorService } from '@services/animator/animator.service';
 export class ThumbnailComponent implements OnInit {
 
   @ViewChild('thumbnail', { static: true }) public thumbnail: ElementRef;
-  @Input() frame: HTMLCanvasElement;
+  @Input() frame: HTMLImageElement;
   @Input() index: number;
   @Output() thumbnailClicked = new EventEmitter();
 
-  public ctx: CanvasRenderingContext2D;
   public width = this.animatorService.animator.width;
   public height = this.animatorService.animator.height;
 
@@ -24,8 +23,7 @@ export class ThumbnailComponent implements OnInit {
   ngOnInit() {
     this.thumbnail.nativeElement.width = this.width;
     this.thumbnail.nativeElement.height = this.height;
-    this.ctx = this.thumbnail.nativeElement.getContext('2d');
-    this.ctx.drawImage(this.frame, 0, 0, this.width, this.height);
+    this.thumbnail.nativeElement.appendChild(this.frame);
   }
 
   onClick() {

src/app/services/video/video.service.ts

@@ -52,14 +52,12 @@ export class VideoService {
       await this.loadFfmpeg();
     }
 
-    // we always use webp - if a png is incoming (e.g. from safari), we'll convert it to webp
+    // we always use webp - if a jpeg is incoming (e.g. from safari), we'll convert it to webp
     const workingDirectory = await this.buildWorkingDirectory()
 
     // write images to the directory in parallel, wait for all images to be stored:
     await this.storeImagesInFilesystem(imageBlobs, workingDirectory)
 
-    console.log("images stored!")
-
     const outputFileName = this.pathToFile(workingDirectory, 'output.mp4')
 
     let parameters = []
@@ -80,64 +78,65 @@ export class VideoService {
     return new Blob([data.buffer], { type: 'video/mp4' });
   }
 
-  // converts a PNG to a webP, which is necessary for the safari export:
-  public async convertPngToWebP(pngs: any[]): Promise<ArrayBuffer[]> {
+  // converts all Jpegs in this list to webP, which is necessary for the safari export:
+  public async convertPotentiallyMixedFrames(potentiallyMixedFrames: any[]): Promise<ArrayBuffer[]> {
     if (!this.loaded) {
       await this.loadFfmpeg();
     }
 
     const workingDirectory = await this.buildWorkingDirectory();
 
-    await Promise.all(pngs.map(async (imageBlob, index) => {
-      return this.ffmpeg.writeFile(this.pathToFile(workingDirectory, `image_${index}.png`), await fetchFile(imageBlob));
-    }));
-
-    await this.convertPngsToWebP(workingDirectory);
-
+    await this.storeImagesInFilesystem(potentiallyMixedFrames, workingDirectory);
+
     let webPs = [];
 
-    for (let i = 0; i < pngs.length; i++) {
-      webPs.push(await this.ffmpeg.readFile(this.pathToFile(workingDirectory, `image_${i+1}.webp`)))
+    for (let i = 0; i < potentiallyMixedFrames.length; i++) {
+      webPs.push(await this.ffmpeg.readFile(this.pathToFile(workingDirectory, `image_${i}.webp`)))
     }
 
     await this.deleteDirectory(workingDirectory);
 
     return webPs;
   }
 
-  private async convertToPngToWebPBatch(pngBlobsWithIndex: {index: number, imageBlob: Blob}[], targetWorkingDirectory: string) {
+  private async convertToJpegToWebPBatch(jpegBlobsWithIndex: {index: number, imageBlob: Blob}[], targetWorkingDirectory: string) {
     if (!this.loaded) {
       await this.loadFfmpeg();
     }
+
+    if (jpegBlobsWithIndex.length < 0) {
+      // nothing to do, all images are already webp
+      return;
+    }
 
     const workingDirectory = await this.buildWorkingDirectory();
 
-    await Promise.all(pngBlobsWithIndex.map(async (imageBlobWithIndex, index) => {
-      return this.ffmpeg.writeFile(this.pathToFile(workingDirectory, `image_${index}.png`), await fetchFile(imageBlobWithIndex.imageBlob));
+    await Promise.all(jpegBlobsWithIndex.map(async (imageBlobWithIndex, index) => {
+      return this.ffmpeg.writeFile(this.pathToFile(workingDirectory, `image_${index}.jpg`), await fetchFile(imageBlobWithIndex.imageBlob));
     }));
 
-    // Unfortunately, ffmpeg does not keep the mapping, e.g. "image_2.png, image_4.png" will not be converted to "image_2.png, image_4.webp", but rather "image_1.webp, image_2.webp"
-    await this.convertPngsToWebP(workingDirectory);
+    // Unfortunately, ffmpeg does not keep the mapping, e.g. "image_2.jpg, image_4.jpg" will not be converted to "image_2.jpg, image_4.webp", but rather "image_1.webp, image_2.webp"
+    await this.convertJpegsToWebP(workingDirectory);
 
     // afterwards, we copy the converted image to the targetworkingdirectory
-    // index follows the consecutive ordering ffmpeg uses, whereas the index from the pngBlobWithIndex is the right one:
-    for (let index = 0; index < pngBlobsWithIndex.length; index++) {
+    // index follows the consecutive ordering ffmpeg uses, whereas the index from the jpegBlobWithIndex is the right one:
+    for (let index = 0; index < jpegBlobsWithIndex.length; index++) {
       const from = this.pathToFile(workingDirectory, `image_${index+1}.webp`)
-      const to = this.pathToFile(targetWorkingDirectory, `image_${pngBlobsWithIndex[index].index}.webp`)
+      const to = this.pathToFile(targetWorkingDirectory, `image_${jpegBlobsWithIndex[index].index}.webp`)
       await this.ffmpeg.writeFile(to, await this.ffmpeg.readFile(from))
     } 
 
     await this.deleteDirectory(workingDirectory)
   }
 
-  private async convertPngsToWebP(workingDirectory: string) {
-    this.ffmpeg.exec(["-i", this.pathToFile(workingDirectory, 'image_%d.png'), "-c:v", "libwebp", "-lossless", "0", "-compression_level", "4", "-quality", "75", this.pathToFile(workingDirectory, 'image_%d.webp')]);
+  private async convertJpegsToWebP(workingDirectory: string) {
+    this.ffmpeg.exec(["-i", this.pathToFile(workingDirectory, 'image_%d.jpg'), "-c:v", "libwebp", "-lossless", "0", "-compression_level", "4", "-quality", "75", this.pathToFile(workingDirectory, 'image_%d.webp')]);
   }
 
   private async storeImagesInFilesystem(imageBlobs: Blob[], workingDirectory: string) {
-    // we can write webps directly, however pngs need to be converted first. we batch the conversion, as otherwise we are likely to get an out of memory error on safari:
+    // we can write webps directly, however jpegs need to be converted first. we batch the conversion, as otherwise we are likely to get an out of memory error on safari:
     let webpBlobsWithIndex = imageBlobs.flatMap((imageBlob, index) => {
-        if (imageBlob.type != MimeTypes.imagePng) {
+        if (imageBlob.type != MimeTypes.imageJpeg) {
           return [{index: index, imageBlob: imageBlob}]
         }
         else {
@@ -146,8 +145,8 @@ export class VideoService {
       }
     )
 
-    let pngBlobsWithIndex = imageBlobs.flatMap((imageBlob, index) => {
-      if (imageBlob.type == MimeTypes.imagePng) {
+    let jpegBlobsWithIndex = imageBlobs.flatMap((imageBlob, index) => {
+      if (imageBlob.type == MimeTypes.imageJpeg) {
         return [{index: index, imageBlob: imageBlob}]
       }
       else {
@@ -160,7 +159,7 @@ export class VideoService {
       return this.ffmpeg.writeFile(this.pathToFile(workingDirectory, `image_${webpBlobWithIndex.index}.webp`), await fetchFile(webpBlobWithIndex.imageBlob));
     }));
 
-    await this.convertToPngToWebPBatch(pngBlobsWithIndex, workingDirectory)
+    await this.convertToJpegToWebPBatch(jpegBlobsWithIndex, workingDirectory)
   }
 
   // ffmpeg can't delete non-empty directories, so we have to delete its content first: