Skip to content

Commit

Permalink
adding conversion
Browse files Browse the repository at this point in the history
  • Loading branch information
pelikhan committed Jan 9, 2025
1 parent 20e4051 commit 921c65e
Show file tree
Hide file tree
Showing 8 changed files with 77 additions and 9 deletions.
5 changes: 3 additions & 2 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
"@octokit/rest": "^21.0.2",
"dockerode": "^4.0.3",
"es-toolkit": "^1.31.0",
"fluent-ffmpeg": "^2.1.3",
"gpt-tokenizer": "^2.8.1",
"html-to-text": "^9.0.5",
"jimp": "^1.6.0",
Expand Down Expand Up @@ -120,9 +121,9 @@
"scripts": {
"compile:runtime": "tsc src/runtime.ts --skipLibCheck --outDir built --declaration --target es2020 --moduleResolution node --module esnext && mv built/runtime.js built/runtime.mjs",
"compile:api": "esbuild src/api.ts --outfile=built/api.mjs",
"compile:cli": "esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema && node ../../scripts/patch-cli.mjs",
"compile:cli": "esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema --external:fluent-ffmpeg && node ../../scripts/patch-cli.mjs",
"compile": "yarn compile:api && yarn compile:runtime && yarn compile:cli",
"compile-debug": "esbuild src/main.ts --sourcemap --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema",
"compile-debug": "esbuild src/main.ts --sourcemap --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema --external:fluent-ffmpeg",
"postcompile": "node built/genaiscript.cjs info help > ../../docs/src/content/docs/reference/cli/commands.md",
"vis:treemap": "npx --yes esbuild-visualizer --metadata esbuild.meta.json --filename esbuild.treemap.html",
"vis:network": "npx --yes esbuild-visualizer --metadata esbuild.meta.json --filename esbuild.network.html --template network",
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/src/runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ export const z = _z
/**
* HuggingFace transformers.js pipeline apis.
*/
export const pipeline = _pipeline
export const pipeline = _pipeline
2 changes: 2 additions & 0 deletions packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
"@smithy/util-base64": "^3.0.0",
"@tidyjs/tidy": "^2.5.2",
"@types/diff": "^6.0.0",
"@types/fluent-ffmpeg": "^2.1.27",
"@types/html-escaper": "^3.0.2",
"@types/html-to-text": "^9.0.4",
"@types/inflection": "^1.13.2",
Expand All @@ -54,6 +55,7 @@
"fetch-retry": "^6.0.0",
"fflate": "^0.8.2",
"file-type": "19.1.1",
"fluent-ffmpeg": "^2.1.3",
"gpt-tokenizer": "^2.8.1",
"groq-js": "^1.14.2",
"html-escaper": "^3.0.3",
Expand Down
43 changes: 43 additions & 0 deletions packages/core/src/ffmpeg.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import { fileTypeFromBuffer } from "file-type"
import { PassThrough } from "stream"
import { logVerbose } from "./util"
import { writeFile } from "fs/promises"
import { Readable } from "stream"

async function importFfmpeg() {
const ffmpeg = await import("fluent-ffmpeg")
return ffmpeg.default
}

export async function convertToAudioBlob(data: Buffer): Promise<Blob> {
const mime = await fileTypeFromBuffer(data)
if (/^audio\//.test(mime?.mime))
return new Blob([data], { type: mime.mime })

logVerbose(`ffmpeg: extracting audio from video...`)
return new Promise<Blob>(async (resolve, reject) => {
const inputStream = new PassThrough()
inputStream.end(data)
await writeFile("input.mp4", data)

console.log(await fileTypeFromBuffer(data))
const outputStream = new PassThrough()
const chunks: Buffer[] = []

outputStream.on("data", (chunk) => chunks.push(chunk))
outputStream.on("end", async () => {
const buffer = Buffer.concat(chunks)
const mime = await fileTypeFromBuffer(buffer)
await writeFile("audio.wav", buffer)
resolve(new Blob([buffer], { type: mime.mime }))
})
outputStream.on("error", reject)

const ffmpeg = await importFfmpeg()
ffmpeg(inputStream)
.toFormat("wav")
.audioBitrate("16k")
.on("error", reject)
.pipe(outputStream)
})
}
9 changes: 4 additions & 5 deletions packages/core/src/runpromptcontext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ import { resolveBufferLike } from "./bufferlike"
import { fileTypeFromBuffer } from "file-type"
import prettyBytes from "pretty-bytes"
import { JSONLineCache } from "./cache"
import { string } from "mathjs"
import { convertToAudioBlob } from "./ffmpeg"

export function createChatTurnGenerationContext(
options: GenerationOptions,
Expand Down Expand Up @@ -666,9 +666,8 @@ export function createChatGenerationContext(
const data = await resolveBufferLike(audio, {
trace: transcriptionTrace,
})
const mimeType = await fileTypeFromBuffer(data)
const file = await convertToAudioBlob(data)
const update: () => Promise<TranscriptionResult> = async () => {
const file = new Blob([data], { type: mimeType.mime })
trace.itemValue(`model`, configuration.model)
trace.itemValue(`file size`, prettyBytes(file.size))
trace.itemValue(`file type`, file.type)
Expand All @@ -690,7 +689,7 @@ export function createChatGenerationContext(

let res: TranscriptionResult
const _cache = JSONLineCache.byName<
{ data: Buffer; type: string } & TranscriptionOptions,
{ file: Blob } & TranscriptionOptions,
TranscriptionResult
>(
cache === true
Expand All @@ -701,7 +700,7 @@ export function createChatGenerationContext(
)
if (cache) {
const hit = await _cache.getOrUpdate(
{ data, type: mimeType.mime, ...rest },
{ file, ...rest },
update,
(res) => !res.error
)
Expand Down
3 changes: 3 additions & 0 deletions packages/sample/genaisrc/transcribe.genai.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,6 @@ for (const segment of res2.segments) {
console.log(`[${start}] ${text}`)
}
console.log(res2)

const resv = await transcribe("src/audio/helloworld.mp4")
console.log(resv)
Binary file added packages/sample/src/audio/helloworld.mp4
Binary file not shown.
22 changes: 21 additions & 1 deletion yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -3234,6 +3234,13 @@
resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.6.tgz#628effeeae2064a1b4e79f78e81d87b7e5fc7b50"
integrity sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==

"@types/fluent-ffmpeg@^2.1.27":
version "2.1.27"
resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.27.tgz#c4eac6fbda30bb6316d2220c8faf54f48db0812d"
integrity sha512-QiDWjihpUhriISNoBi2hJBRUUmoj/BMTYcfz+F+ZM9hHWBYABFAE6hjP/TbCZC0GWwlpa3FzvHH9RzFeRusZ7A==
dependencies:
"@types/node" "*"

"@types/fs-extra@>=11", "@types/fs-extra@^11.0.4":
version "11.0.4"
resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.4.tgz#e16a863bb8843fba8c5004362b5a73e17becca45"
Expand Down Expand Up @@ -3826,6 +3833,11 @@ assert@^2.1.0:
object.assign "^4.1.4"
util "^0.12.5"

async@^0.2.9:
version "0.2.10"
resolved "https://registry.yarnpkg.com/async/-/async-0.2.10.tgz#b6bbe0b0674b9d719708ca38de8c237cb526c3d1"
integrity sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==

asynckit@^0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
Expand Down Expand Up @@ -5551,6 +5563,14 @@ flatbuffers@^1.12.0:
resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-1.12.0.tgz#72e87d1726cb1b216e839ef02658aa87dcef68aa"
integrity sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==

fluent-ffmpeg@^2.1.3:
version "2.1.3"
resolved "https://registry.yarnpkg.com/fluent-ffmpeg/-/fluent-ffmpeg-2.1.3.tgz#d6846be257777844249a4adeb320f25326d239f3"
integrity sha512-Be3narBNt2s6bsaqP6Jzq91heDgOEaDCJAXcE3qcma/EJBSy5FB4cvO31XBInuAuKBx8Kptf8dkhjK0IOru39Q==
dependencies:
async "^0.2.9"
which "^1.1.1"

follow-redirects@^1.14.8, follow-redirects@^1.15.6:
version "1.15.9"
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1"
Expand Down Expand Up @@ -10800,7 +10820,7 @@ which-typed-array@^1.1.16, which-typed-array@^1.1.18, which-typed-array@^1.1.2:
gopd "^1.2.0"
has-tostringtag "^1.0.2"

which@^1.2.9:
which@^1.1.1, which@^1.2.9:
version "1.3.1"
resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
Expand Down

0 comments on commit 921c65e

Please sign in to comment.