From 66060c8c45ffd6c72f022efc1b94ec736a55f970 Mon Sep 17 00:00:00 2001 From: nwittstruck Date: Fri, 10 May 2024 15:32:52 +0200 Subject: [PATCH] bug: In safari the canvas uses too much memory #61 --- .github/workflows/on_release_build_image.yml | 2 +- src/app/enums/mime-types.enum.ts | 2 +- src/app/models/animator.ts | 72 ++++++++++++------- .../thumbnail/thumbnail.component.html | 4 +- .../thumbnail/thumbnail.component.ts | 6 +- src/app/services/video/video.service.ts | 55 +++++++------- 6 files changed, 79 insertions(+), 62 deletions(-) diff --git a/.github/workflows/on_release_build_image.yml b/.github/workflows/on_release_build_image.yml index 4995740..da5d3b7 100644 --- a/.github/workflows/on_release_build_image.yml +++ b/.github/workflows/on_release_build_image.yml @@ -6,7 +6,7 @@ name: Create and publish a Docker image on: push: - branches: ["main"] + branches: ["main", "61-bug-in-safari-the-canvas-uses-too-much-memory"] release: types: [published] diff --git a/src/app/enums/mime-types.enum.ts b/src/app/enums/mime-types.enum.ts index e4385f9..dafdccd 100644 --- a/src/app/enums/mime-types.enum.ts +++ b/src/app/enums/mime-types.enum.ts @@ -4,5 +4,5 @@ export enum MimeTypes { audioWebm = 'audio/webm;codecs=opus', audioMp4 = 'audio/mp4;codecs=mp4a.67', audioMp3 = 'audio/mpeg', - imagePng = 'image/png' + imageJpeg = 'image/jpeg' } diff --git a/src/app/models/animator.ts b/src/app/models/animator.ts index 94bdff6..d512021 100644 --- a/src/app/models/animator.ts +++ b/src/app/models/animator.ts @@ -40,6 +40,8 @@ export class Animator { videoStream: MediaStream; width: any; zeroPlayTime: number; + imageCanvas: any; + context: any; private isAnimatorPlaying: BehaviorSubject; private frameRate: BehaviorSubject; @@ -135,6 +137,8 @@ export class Animator { this.videoStream = stream; this.isStreaming = true; + this.setupContext() + return stream; } catch (err) { console.error(err); @@ -142,29 +146,46 @@ export class Animator { } } + private setupContext() { + this.imageCanvas = document.createElement('canvas'); + this.imageCanvas.id = "capture-from-video-canvas" + this.imageCanvas.width = this.width; + this.imageCanvas.height = this.height; + + this.context = this.imageCanvas.getContext('2d', { alpha: false }); + if (this.rotated) { + this.context.rotate(Math.PI); + this.context.translate(-this.width, -this.height); + } + } + /* * Method is used to capture new image and create canvas out of it and share it with other components */ public async capture() { // console.log('🚀 ~ file: animator.ts ~ line 103 ~ Animator ~ capture ~ capture'); if (!this.isStreaming) { return; } - const imageCanvas: HTMLCanvasElement = document.createElement('canvas'); - imageCanvas.width = this.width; - imageCanvas.height = this.height; - const context = imageCanvas.getContext('2d', { alpha: false }); - if (this.rotated) { - context.rotate(Math.PI); - context.translate(-this.width, -this.height); - } - context.drawImage(this.video, 0, 0, this.width, this.height); - this.frames.push(imageCanvas); + + this.context.drawImage(this.video, 0, 0, this.width, this.height); + + this.imageCanvas.toBlob((blob: Blob) => { + var img = new Image(); + + const dataUrl = URL.createObjectURL(blob); + + img.onload = async() => { + this.frames.push(img); + URL.revokeObjectURL(dataUrl); + } + img.src = dataUrl; + + this.snapshotContext.clearRect(0, 0, this.width, this.height); + this.snapshotContext.drawImage(this.imageCanvas, 0, 0, this.width, this.height); + + this.frameWebps.push(blob); + + }, 'image/jpeg', 0.8); - const promise = await new Promise(((resolve, reject) => { - this.snapshotContext.clearRect(0, 0, this.width, this.height); - this.snapshotContext.drawImage(imageCanvas, 0, 0, this.width, this.height); - imageCanvas.toBlob(blob => { resolve(blob); }, 'image/webp'); - })); - this.frameWebps.push(promise); return this.frames; } @@ -448,16 +469,18 @@ export class Animator { // If not specified this defaults to the same value as `quality`. }); - if (this.isSafari()) { - const convertedFrames = await this.videoService.convertPngToWebP(this.frameWebps); + // ::TODO:: in the past, we only converted the frames for Safari. Right now, we always create jpegs. + // This simplifies the process quite a bit, as it's more consistent. However, we should check if this causes further issues. + //if (this.isSafari()) { + const convertedFrames = await this.videoService.convertPotentiallyMixedFrames(this.frameWebps); for (const frame of convertedFrames) { videoWriter.addFrame(this.uint8ToBase64(frame)); } - } else { + /*} else { for (const frame of this.frames) { videoWriter.addFrame(frame); } - } + }*/ const blob = await videoWriter.complete(); return blob; @@ -551,7 +574,7 @@ export class Animator { */ private addFrameVP8(frameOffset: number, callback: any, blob: Blob, index: number) { let blobURL = URL.createObjectURL(blob); - const image = new Image(this.width, this.height); + const image = new Image(); this.framesInFlight++; image.addEventListener('error', (error) => { if (image.getAttribute('triedvp8l')) { @@ -571,11 +594,8 @@ export class Animator { }); image.addEventListener('load', async (evt: any) => { - const newCanvas = document.createElement('canvas'); - newCanvas.width = this.width; - newCanvas.height = this.height; - newCanvas.getContext('2d', { alpha: false }).drawImage(evt.target, 0, 0, this.width, this.height); - this.frames[frameOffset + index] = newCanvas; + this.frames[frameOffset + index] = image + this.frameWebps[frameOffset + index] = await new Promise((resolve, reject) => { resolve(blob); }); diff --git a/src/app/pages/animator/components/thumbnail/thumbnail.component.html b/src/app/pages/animator/components/thumbnail/thumbnail.component.html index d675318..d911b5e 100644 --- a/src/app/pages/animator/components/thumbnail/thumbnail.component.html +++ b/src/app/pages/animator/components/thumbnail/thumbnail.component.html @@ -1,4 +1,4 @@
- +
-
+ \ No newline at end of file diff --git a/src/app/pages/animator/components/thumbnail/thumbnail.component.ts b/src/app/pages/animator/components/thumbnail/thumbnail.component.ts index aef22ea..d3390d9 100644 --- a/src/app/pages/animator/components/thumbnail/thumbnail.component.ts +++ b/src/app/pages/animator/components/thumbnail/thumbnail.component.ts @@ -9,11 +9,10 @@ import { AnimatorService } from '@services/animator/animator.service'; export class ThumbnailComponent implements OnInit { @ViewChild('thumbnail', { static: true }) public thumbnail: ElementRef; - @Input() frame: HTMLCanvasElement; + @Input() frame: HTMLImageElement; @Input() index: number; @Output() thumbnailClicked = new EventEmitter(); - public ctx: CanvasRenderingContext2D; public width = this.animatorService.animator.width; public height = this.animatorService.animator.height; @@ -24,8 +23,7 @@ export class ThumbnailComponent implements OnInit { ngOnInit() { this.thumbnail.nativeElement.width = this.width; this.thumbnail.nativeElement.height = this.height; - this.ctx = this.thumbnail.nativeElement.getContext('2d'); - this.ctx.drawImage(this.frame, 0, 0, this.width, this.height); + this.thumbnail.nativeElement.appendChild(this.frame); } onClick() { diff --git a/src/app/services/video/video.service.ts b/src/app/services/video/video.service.ts index 9c13dc0..eb7de0c 100644 --- a/src/app/services/video/video.service.ts +++ b/src/app/services/video/video.service.ts @@ -52,14 +52,12 @@ export class VideoService { await this.loadFfmpeg(); } - // we always use webp - if a png is incoming (e.g. from safari), we'll convert it to webp + // we always use webp - if a jpeg is incoming (e.g. from safari), we'll convert it to webp const workingDirectory = await this.buildWorkingDirectory() // write images to the directory in parallel, wait for all images to be stored: await this.storeImagesInFilesystem(imageBlobs, workingDirectory) - console.log("images stored!") - const outputFileName = this.pathToFile(workingDirectory, 'output.mp4') let parameters = [] @@ -80,24 +78,20 @@ export class VideoService { return new Blob([data.buffer], { type: 'video/mp4' }); } - // converts a PNG to a webP, which is necessary for the safari export: - public async convertPngToWebP(pngs: any[]): Promise { + // converts all Jpegs in this list to webP, which is necessary for the safari export: + public async convertPotentiallyMixedFrames(potentiallyMixedFrames: any[]): Promise { if (!this.loaded) { await this.loadFfmpeg(); } const workingDirectory = await this.buildWorkingDirectory(); - await Promise.all(pngs.map(async (imageBlob, index) => { - return this.ffmpeg.writeFile(this.pathToFile(workingDirectory, `image_${index}.png`), await fetchFile(imageBlob)); - })); - - await this.convertPngsToWebP(workingDirectory); - + await this.storeImagesInFilesystem(potentiallyMixedFrames, workingDirectory); + let webPs = []; - for (let i = 0; i < pngs.length; i++) { - webPs.push(await this.ffmpeg.readFile(this.pathToFile(workingDirectory, `image_${i+1}.webp`))) + for (let i = 0; i < potentiallyMixedFrames.length; i++) { + webPs.push(await this.ffmpeg.readFile(this.pathToFile(workingDirectory, `image_${i}.webp`))) } await this.deleteDirectory(workingDirectory); @@ -105,39 +99,44 @@ export class VideoService { return webPs; } - private async convertToPngToWebPBatch(pngBlobsWithIndex: {index: number, imageBlob: Blob}[], targetWorkingDirectory: string) { + private async convertToJpegToWebPBatch(jpegBlobsWithIndex: {index: number, imageBlob: Blob}[], targetWorkingDirectory: string) { if (!this.loaded) { await this.loadFfmpeg(); } + + if (jpegBlobsWithIndex.length < 0) { + // nothing to do, all images are already webp + return; + } const workingDirectory = await this.buildWorkingDirectory(); - await Promise.all(pngBlobsWithIndex.map(async (imageBlobWithIndex, index) => { - return this.ffmpeg.writeFile(this.pathToFile(workingDirectory, `image_${index}.png`), await fetchFile(imageBlobWithIndex.imageBlob)); + await Promise.all(jpegBlobsWithIndex.map(async (imageBlobWithIndex, index) => { + return this.ffmpeg.writeFile(this.pathToFile(workingDirectory, `image_${index}.jpg`), await fetchFile(imageBlobWithIndex.imageBlob)); })); - // Unfortunately, ffmpeg does not keep the mapping, e.g. "image_2.png, image_4.png" will not be converted to "image_2.png, image_4.webp", but rather "image_1.webp, image_2.webp" - await this.convertPngsToWebP(workingDirectory); + // Unfortunately, ffmpeg does not keep the mapping, e.g. "image_2.jpg, image_4.jpg" will not be converted to "image_2.jpg, image_4.webp", but rather "image_1.webp, image_2.webp" + await this.convertJpegsToWebP(workingDirectory); // afterwards, we copy the converted image to the targetworkingdirectory - // index follows the consecutive ordering ffmpeg uses, whereas the index from the pngBlobWithIndex is the right one: - for (let index = 0; index < pngBlobsWithIndex.length; index++) { + // index follows the consecutive ordering ffmpeg uses, whereas the index from the jpegBlobWithIndex is the right one: + for (let index = 0; index < jpegBlobsWithIndex.length; index++) { const from = this.pathToFile(workingDirectory, `image_${index+1}.webp`) - const to = this.pathToFile(targetWorkingDirectory, `image_${pngBlobsWithIndex[index].index}.webp`) + const to = this.pathToFile(targetWorkingDirectory, `image_${jpegBlobsWithIndex[index].index}.webp`) await this.ffmpeg.writeFile(to, await this.ffmpeg.readFile(from)) } await this.deleteDirectory(workingDirectory) } - private async convertPngsToWebP(workingDirectory: string) { - this.ffmpeg.exec(["-i", this.pathToFile(workingDirectory, 'image_%d.png'), "-c:v", "libwebp", "-lossless", "0", "-compression_level", "4", "-quality", "75", this.pathToFile(workingDirectory, 'image_%d.webp')]); + private async convertJpegsToWebP(workingDirectory: string) { + this.ffmpeg.exec(["-i", this.pathToFile(workingDirectory, 'image_%d.jpg'), "-c:v", "libwebp", "-lossless", "0", "-compression_level", "4", "-quality", "75", this.pathToFile(workingDirectory, 'image_%d.webp')]); } private async storeImagesInFilesystem(imageBlobs: Blob[], workingDirectory: string) { - // we can write webps directly, however pngs need to be converted first. we batch the conversion, as otherwise we are likely to get an out of memory error on safari: + // we can write webps directly, however jpegs need to be converted first. we batch the conversion, as otherwise we are likely to get an out of memory error on safari: let webpBlobsWithIndex = imageBlobs.flatMap((imageBlob, index) => { - if (imageBlob.type != MimeTypes.imagePng) { + if (imageBlob.type != MimeTypes.imageJpeg) { return [{index: index, imageBlob: imageBlob}] } else { @@ -146,8 +145,8 @@ export class VideoService { } ) - let pngBlobsWithIndex = imageBlobs.flatMap((imageBlob, index) => { - if (imageBlob.type == MimeTypes.imagePng) { + let jpegBlobsWithIndex = imageBlobs.flatMap((imageBlob, index) => { + if (imageBlob.type == MimeTypes.imageJpeg) { return [{index: index, imageBlob: imageBlob}] } else { @@ -160,7 +159,7 @@ export class VideoService { return this.ffmpeg.writeFile(this.pathToFile(workingDirectory, `image_${webpBlobWithIndex.index}.webp`), await fetchFile(webpBlobWithIndex.imageBlob)); })); - await this.convertToPngToWebPBatch(pngBlobsWithIndex, workingDirectory) + await this.convertToJpegToWebPBatch(jpegBlobsWithIndex, workingDirectory) } // ffmpeg can't delete non-empty directories, so we have to delete its content first: