Merge pull request #74 from izkizk8/thumbnails

feat(av-cliper): MP4Clip thumbnails support geting by interval

doc-site/docs/demo/6_1-video-thumbnails.md

@@ -63,3 +63,60 @@ export default function UI() {
   );
 }
 ```
+
+## 自定义步长
+
+可以通过指定视频时间段、步长（单位微秒）来获取视频帧。该 DEMO 从视频 10s - 60s 之间每隔 10s 抽取一帧。
+
+```tsx
+import { useState, useEffect } from 'react';
+import { MP4Clip } from '@webav/av-cliper';
+import { assetsPrefix } from './utils';
+
+const resList = assetsPrefix(['video/bunny.mp4']);
+
+async function start() {
+  const clip = new MP4Clip((await fetch(resList[0])).body!);
+  await clip.ready;
+  return await clip.thumbnails(200, { start: 10e6, end: 60e6, step: 10e6 });
+}
+
+export default function UI() {
+  const [imgList, setImgList] = useState<Array<{ ts: number; img: string }>>(
+    [],
+  );
+  const [cost, setCost] = useState(0);
+
+  useEffect(() => {
+    let startTime = performance.now();
+    (async () => {
+      setImgList(
+        (await start()).map((it) => ({
+          ts: it.ts,
+          img: URL.createObjectURL(it.img),
+        })),
+      );
+      setCost(((performance.now() - startTime) / 1000).toFixed(2));
+    })();
+  }, []);
+
+  return (
+    <>
+      <div>
+        {imgList.length === 0
+          ? 'loading...'
+          : `耗时：${cost}s，提取帧数：${imgList.length}`}
+      </div>
+      <br />
+      <div className="flex flex-wrap">
+        {imgList.map((it) => (
+          <div key={it.ts}>
+            <div className="text-center">{(it.ts / 1e6).toFixed(2)}s</div>
+            <img src={it.img}></img>
+          </div>
+        ))}
+      </div>
+    </>
+  );
+}
+```

packages/av-cliper/src/clips/__tests__/mp4-clip.test.ts

@@ -65,8 +65,12 @@ test('thumbnails', async () => {
   const thumbnails = await clip.thumbnails();
   expect(thumbnails.length).toBe(9);
   expect((await createImageBitmap(thumbnails[0].img)).width).toBe(100);
-  const thumbnails150 = await clip.thumbnails(150);
-  expect(thumbnails150.length).toBe(9);
+  const thumbnails150 = await clip.thumbnails(150, {
+    start: 1e6,
+    end: 10e6,
+    step: 1e6,
+  });
+  expect(thumbnails150.length).toBe(10);
   expect((await createImageBitmap(thumbnails150[0].img)).width).toBe(150);
   clip.destroy();
 });

packages/av-cliper/src/clips/mp4-clip.ts

@@ -242,7 +242,17 @@ export class MP4Clip implements IClip {
     }
   }
 
-  thumbnails(imgWidth = 100): Promise<Array<{ ts: number; img: Blob }>> {
+  /**
+   * Generate thumbnails, default generate 100px width thumbnails by every key frame.
+   *
+   * @param imgWidth thumbnail width, default 100
+   * @param opts Partial<ThumbnailOpts>
+   * @returns Promise<Array<{ ts: number; img: Blob }>>
+   */
+  thumbnails(
+    imgWidth = 100,
+    opts?: Partial<ThumbnailOpts>,
+  ): Promise<Array<{ ts: number; img: Blob }>> {
     const vc = this.#decoderConf.video;
     if (vc == null) return Promise.resolve([]);
 
@@ -266,31 +276,60 @@ export class MP4Clip implements IClip {
         );
       }
 
-      const samples = this.#videoSamples
-        .filter((s) => !s.deleted && s.is_sync)
-        .map(sample2VideoChunk);
-      if (samples.length === 0) {
-        resolver();
-        return;
+      function pushPngPromise(vf: VideoFrame) {
+        pngPromises.push({
+          ts: vf.timestamp,
+          img: convtr(vf),
+        });
       }
 
-      let cnt = 0;
-      const dec = new VideoDecoder({
-        output: (vf) => {
-          cnt += 1;
-          pngPromises.push({
-            ts: vf.timestamp,
-            img: convtr(vf),
-          });
-          if (cnt === samples.length) resolver();
-        },
-        error: Log.error,
-      });
-      dec.configure(vc);
-      samples.forEach((c) => {
-        dec.decode(c);
-      });
-      await dec.flush();
+      const { start = 0, end = this.#meta.duration, step } = opts ?? {};
+      if (step) {
+        if (
+          this.#decoderConf.video == null ||
+          this.#videoSamples.length === 0
+        ) {
+          resolver();
+          return;
+        }
+        let cur = start;
+        // 创建一个新的 VideoFrameFinder 实例，避免与 tick 方法共用而导致冲突
+        const videoFrameFinder = new VideoFrameFinder(
+          this.#videoSamples,
+          this.#decoderConf.video,
+        );
+        while (cur <= end) {
+          const vf = await videoFrameFinder.find(cur);
+          if (vf) pushPngPromise(vf);
+          cur += step;
+        }
+        resolver();
+      } else {
+        const samples = this.#videoSamples
+          .filter(
+            (s) => !s.deleted && s.is_sync && s.cts >= start && s.cts <= end,
+          )
+          .map(sample2VideoChunk);
+        if (samples.length === 0) {
+          resolver();
+          return;
+        }
+
+        let cnt = 0;
+        const dec = new VideoDecoder({
+          output: (vf) => {
+            cnt += 1;
+            pushPngPromise(vf);
+            if (cnt === samples.length) resolver();
+          },
+          error: Log.error,
+        });
+        dec.configure(vc);
+        samples.forEach((c) => {
+          dec.decode(c);
+        });
+        await dec.flush();
+      }
     });
   }
 
@@ -786,3 +825,9 @@ function createVF2BlobConvtr(
     return blob;
   };
 }
+
+export type ThumbnailOpts = {
+  start: number;
+  end: number;
+  step: number;
+};