diff --git a/.vscode/launch.json b/.vscode/launch.json index 738c9d8..ccda5bb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -75,6 +75,29 @@ "--mode=development" ], }, + { + "name": "index-docs", + "type": "node", + "request": "launch", + "program": "${workspaceFolder}/dist-util/index-docs.mjs", + "args": [ + "--dry-run", + "../HPCC-Platform/docs" + ], + "outFiles": [ + "${workspaceFolder}/dist-util/**/*.js" + ] + }, + { + "name": "retrieve-docs", + "type": "node", + "request": "launch", + "program": "${workspaceFolder}/dist-util/retrieve-docs.mjs", + "args": [], + "outFiles": [ + "${workspaceFolder}/dist-util/**/*.js" + ] + }, { "name": "Launch Server Adapter", "type": "node", diff --git a/esbuild.mjs b/esbuild.mjs index e6d0a66..777b8a8 100644 --- a/esbuild.mjs +++ b/esbuild.mjs @@ -2,13 +2,13 @@ import * as esbuild from "esbuild"; import copyStaticFiles from "esbuild-copy-static-files"; import process from "node:process"; import path from "node:path"; -import { problemMatcher, removeStrict } from "@hpcc-js/esbuild-plugins"; +import { problemMatcher, removeStrict, nodeTpl } from "@hpcc-js/esbuild-plugins"; import tsconfigNode from "./tsconfig.json" with {"type": "json"}; import tsconfigBrowser from "./tsconfig.webview.json" with {"type": "json"}; const outputDirectory = "dist"; -const production = process.argv.includes("--production"); const watch = process.argv.includes("--watch"); +const production = !watch && process.argv.includes("--production"); async function main(tsconfigRaw, entryPoint, platform, format, plugins = []) { const ctx = await esbuild.context({ @@ -41,12 +41,16 @@ Promise.all([ copyStaticFiles({ src: "./node_modules/@hpcc-js/ddl-shim/schema/v2.json", dest: path.join(outputDirectory, "v2.json"), + }), copyStaticFiles({ + src: "./util/docs.vecdb", + dest: path.join(outputDirectory, "docs.vecdb"), }) ]), main(tsconfigBrowser, "./src/notebook/renderers/wuRenderer.tsx", "browser", "esm"), main(tsconfigBrowser, "./src/notebook/renderers/ojsRenderer.ts", "browser", "esm"), main(tsconfigBrowser, "./src/eclwatch.tsx", "browser", "iife", [removeStrict()]), - main(tsconfigBrowser, "./src/web-extension.ts", "browser", "iife") + main(tsconfigBrowser, "./src/web-extension.ts", "browser", "iife"), + nodeTpl("./util/index-docs.ts", "./dist-util/index-docs", "esm") ]).catch((e) => { console.error(e); process.exit(1); diff --git a/package-lock.json b/package-lock.json index 2970d7b..bea4516 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,42 +8,48 @@ "name": "ecl", "version": "2.28.2", "dependencies": { - "@vscode/prompt-tsx": "0.2.6-alpha", + "@vscode/prompt-tsx": "0.2.8-alpha", "exceljs": "4.4.0", - "html-to-text": "^9.0.5" + "html-to-text": "9.0.5" }, "devDependencies": { - "@eslint/js": "9.9.0", - "@fluentui/react": "8.120.6", + "@eslint/js": "9.9.1", + "@fluentui/react": "8.120.7", "@hpcc-js/common": "2.72.0", "@hpcc-js/comms": "2.95.0", "@hpcc-js/ddl-shim": "2.21.0", "@hpcc-js/dgrid2": "2.4.0", - "@hpcc-js/esbuild-plugins": "1.1.0", - "@hpcc-js/loader": "2.106.4", + "@hpcc-js/esbuild-plugins": "1.1.2", + "@hpcc-js/loader": "2.106.7", "@hpcc-js/marshaller": "2.26.4", "@hpcc-js/observable-shim": "2.6.0", "@hpcc-js/observablehq-compiler": "1.3.0", "@hpcc-js/util": "2.52.0", + "@hpcc-js/wasm-expat": "1.3.0", + "@hpcc-js/wasm-llama": "1.0.0", + "@hpcc-js/wasm-zstd": "1.2.0", + "@langchain/community": "0.2.32", + "@langchain/core": "0.2.31", + "@langchain/ollama": "0.0.4", "@observablehq/inspector": "5.0.1", "@observablehq/parser": "6.1.0", "@observablehq/runtime": "5.9.9", "@observablehq/stdlib": "5.8.8", "@octokit/rest": "21.0.2", "@types/mocha": "10.0.7", - "@types/node": "20.16.1", + "@types/node": "20.16.5", "@types/react": "17.0.80", "@types/react-dom": "17.0.25", "@types/tmp": "0.2.6", "@types/vscode": "1.92.0", "@types/vscode-notebook-renderer": "1.72.3", - "@typescript-eslint/eslint-plugin": "8.2.0", - "@typescript-eslint/parser": "8.2.0", - "@vscode/debugadapter": "1.66.0", - "@vscode/debugprotocol": "1.66.0", + "@typescript-eslint/eslint-plugin": "8.4.0", + "@typescript-eslint/parser": "8.4.0", + "@vscode/debugadapter": "1.67.0", + "@vscode/debugprotocol": "1.67.0", "@vscode/extension-telemetry": "0.9.7", - "@vscode/vsce": "3.0.0", - "adm-zip": "0.5.15", + "@vscode/vsce": "3.1.0", + "adm-zip": "0.5.16", "antlr4": "4.13.2", "assert": "2.1.0", "copy-to-clipboard": "3.3.3", @@ -56,17 +62,20 @@ "esbuild-plugin-umd-wrapper": "3.0.0", "eslint": "8.57.0", "eslint-plugin-react-hooks": "4.6.2", + "langchain": "0.2.18", "mocha": "10.7.3", "npm-run-all": "4.1.5", "react": "17.0.2", "react-dom": "17.0.2", "rimraf": "6.0.1", + "run-script-os": "^1.1.6", "standard-version": "9.5.0", - "tslib": "2.6.3", + "tslib": "2.7.0", "typescript": "5.5.4", - "typescript-eslint": "8.2.0", + "typescript-eslint": "8.4.0", "use-resize-observer": "9.1.0", - "uuid": "10.0.0" + "uuid": "10.0.0", + "yargs": "^17.7.2" }, "engines": { "vscode": "^1.92.0" @@ -888,9 +897,9 @@ } }, "node_modules/@eslint/js": { - "version": "9.9.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.9.0.tgz", - "integrity": "sha512-hhetes6ZHP3BlXLxmd8K2SNgkhNSi+UcecbnwWKwpP7kyi/uC75DJ1lOOBO3xrC4jyojtGE3YxKZPHfk4yrgug==", + "version": "9.9.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.9.1.tgz", + "integrity": "sha512-xIDQRsfg5hNBqHz04H1R3scSVwmI+KUbqjsQKHKQ1DAUSaUjYPReZZmS/5PNiKu1fUvzDd6H7DEDKACSEhu+TQ==", "dev": true, "license": "MIT", "engines": { @@ -1023,9 +1032,9 @@ } }, "node_modules/@fluentui/react": { - "version": "8.120.6", - "resolved": "https://registry.npmjs.org/@fluentui/react/-/react-8.120.6.tgz", - "integrity": "sha512-iW4AfHAOtMXlnZ4mDY3eYGlnKLuQrLF5lFt9EN9+A8Jw8Z4LVWoxA0MpSDmcPRDLxU4JQ9ukoICwYQ1YeS+zlw==", + "version": "8.120.7", + "resolved": "https://registry.npmjs.org/@fluentui/react/-/react-8.120.7.tgz", + "integrity": "sha512-eU8b3X6t3v6Gv38vYXXnI9JBnaTY1wpeLmErPcdBIuvVpAq35/HU+FmR1CB6FDd9oOQqC4sbM6nTlI6mSudmsw==", "dev": true, "license": "MIT", "dependencies": { @@ -1346,15 +1355,24 @@ } }, "node_modules/@hpcc-js/esbuild-plugins": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@hpcc-js/esbuild-plugins/-/esbuild-plugins-1.1.0.tgz", - "integrity": "sha512-GjzHpq7dANSJUUQT9JIBNN1f/hKaIjHfp6PQG6GQt3csTmiHt/q2daFB479taPbLwfR43O5TAeZmqCVQRBuk6g==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@hpcc-js/esbuild-plugins/-/esbuild-plugins-1.1.2.tgz", + "integrity": "sha512-M4RaTZJQDNltgSLfeGWh0Y+XO8wwn7Z4oMWy4LGNQ8otABB99jyNt9GilaAPt6DF5pDIGMxV/gp/VrcFvFJ2zA==", "dev": true, "license": "Apache-2.0", "dependencies": { "@hpcc-js/wasm-base91": "1.0.1", "@hpcc-js/wasm-zstd": "1.0.1", - "fzstd": "0.1.1", + "fzstd": "0.1.1" + } + }, + "node_modules/@hpcc-js/esbuild-plugins/node_modules/@hpcc-js/wasm-zstd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/wasm-zstd/-/wasm-zstd-1.0.1.tgz", + "integrity": "sha512-at8AGwzQVdyg6vSHX85HW1FtEeqU+mo7kzH3nsiWRi9QPyl0s2uu7YXkR9zCq6cEP0uPKImo3ip3xRAkPzNizA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { "yargs": "17.7.2" } }, @@ -1421,9 +1439,9 @@ } }, "node_modules/@hpcc-js/loader": { - "version": "2.106.4", - "resolved": "https://registry.npmjs.org/@hpcc-js/loader/-/loader-2.106.4.tgz", - "integrity": "sha512-sI1RARi/2xbawgJPvBwXznCwQQyEy0GW7v3O2CgYk7ZbIKbK8U2iSf/Nj/sf0TyjceMrweDPpggEqHY7dBqh8A==", + "version": "2.106.7", + "resolved": "https://registry.npmjs.org/@hpcc-js/loader/-/loader-2.106.7.tgz", + "integrity": "sha512-TvJ85L+lccv9Tas19ZQRiKXu8pEjwEu5Mif1/FvPif1TUN+Oaz5FelEgnlUGlj5gI3N+PRHw1zkRQ6tw3loVUQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -1640,6 +1658,13 @@ "tslib": "2.6.3" } }, + "node_modules/@hpcc-js/util/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", + "dev": true, + "license": "0BSD" + }, "node_modules/@hpcc-js/wasm": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/@hpcc-js/wasm/-/wasm-1.20.1.tgz", @@ -1663,15 +1688,26 @@ "yargs": "17.7.2" } }, + "node_modules/@hpcc-js/wasm-expat": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/wasm-expat/-/wasm-expat-1.3.0.tgz", + "integrity": "sha512-i7yz5xlckm9RnBU5kJd4WDamHYMG7BuwKtBBqWgu45ZS1qIzRrSoILqcpjfrVUlzjDD6sLlLIM/F8IdPEEnc4Q==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@hpcc-js/wasm-llama": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/wasm-llama/-/wasm-llama-1.0.0.tgz", + "integrity": "sha512-NJyH0BJu1Sm5GRmVBXrX4R9y97ev2zDNFX28czES4sjIpJEG9oInybpS5Hn+A7KiiABrH4kZcZ1FvwIt6nBRLg==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/@hpcc-js/wasm-zstd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@hpcc-js/wasm-zstd/-/wasm-zstd-1.0.1.tgz", - "integrity": "sha512-at8AGwzQVdyg6vSHX85HW1FtEeqU+mo7kzH3nsiWRi9QPyl0s2uu7YXkR9zCq6cEP0uPKImo3ip3xRAkPzNizA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/wasm-zstd/-/wasm-zstd-1.2.0.tgz", + "integrity": "sha512-Df4PyOPHM2ExQxPUxunI7R55jBQ9wHrwdPO/PBlUIRQdK/oRx4cYu8V1nFPU1wEVca14+Tku48ZTu8WifU+cAw==", "dev": true, - "license": "Apache-2.0", - "dependencies": { - "yargs": "17.7.2" - } + "license": "Apache-2.0" }, "node_modules/@hpcc-js/wasm/node_modules/yargs": { "version": "17.6.0", @@ -1884,6 +1920,690 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/@langchain/community": { + "version": "0.2.32", + "resolved": "https://registry.npmjs.org/@langchain/community/-/community-0.2.32.tgz", + "integrity": "sha512-FKaanuu+QAq5gRVEhCAW/CKP1cd+QG3oKY2AaL/S0yQGnCni0Q5E+YC1m6OSm6hcCv5bX7Gw6prQDJOPi+xrpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@langchain/core": ">=0.2.21 <0.3.0", + "@langchain/openai": ">=0.2.0 <0.3.0", + "binary-extensions": "^2.2.0", + "expr-eval": "^2.0.2", + "flat": "^5.0.2", + "js-yaml": "^4.1.0", + "langchain": "~0.2.3", + "langsmith": "~0.1.30", + "uuid": "^10.0.0", + "zod": "^3.22.3", + "zod-to-json-schema": "^3.22.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@aws-crypto/sha256-js": "^5.0.0", + "@aws-sdk/client-bedrock-agent-runtime": "^3.583.0", + "@aws-sdk/client-bedrock-runtime": "^3.422.0", + "@aws-sdk/client-dynamodb": "^3.310.0", + "@aws-sdk/client-kendra": "^3.352.0", + "@aws-sdk/client-lambda": "^3.310.0", + "@aws-sdk/client-s3": "^3.310.0", + "@aws-sdk/client-sagemaker-runtime": "^3.310.0", + "@aws-sdk/client-sfn": "^3.310.0", + "@aws-sdk/credential-provider-node": "^3.388.0", + "@azure/search-documents": "^12.0.0", + "@azure/storage-blob": "^12.15.0", + "@browserbasehq/sdk": "*", + "@clickhouse/client": "^0.2.5", + "@cloudflare/ai": "*", + "@datastax/astra-db-ts": "^1.0.0", + "@elastic/elasticsearch": "^8.4.0", + "@getmetal/metal-sdk": "*", + "@getzep/zep-cloud": "^1.0.6", + "@getzep/zep-js": "^0.9.0", + "@gomomento/sdk": "^1.51.1", + "@gomomento/sdk-core": "^1.51.1", + "@google-ai/generativelanguage": "*", + "@google-cloud/storage": "^6.10.1 || ^7.7.0", + "@gradientai/nodejs-sdk": "^1.2.0", + "@huggingface/inference": "^2.6.4", + "@langchain/langgraph": "*", + "@layerup/layerup-security": "^1.5.12", + "@mendable/firecrawl-js": "^0.0.13", + "@mlc-ai/web-llm": "0.2.46", + "@mozilla/readability": "*", + "@neondatabase/serverless": "*", + "@notionhq/client": "^2.2.10", + "@opensearch-project/opensearch": "*", + "@pinecone-database/pinecone": "*", + "@planetscale/database": "^1.8.0", + "@premai/prem-sdk": "^0.3.25", + "@qdrant/js-client-rest": "^1.8.2", + "@raycast/api": "^1.55.2", + "@rockset/client": "^0.9.1", + "@smithy/eventstream-codec": "^2.0.5", + "@smithy/protocol-http": "^3.0.6", + "@smithy/signature-v4": "^2.0.10", + "@smithy/util-utf8": "^2.0.0", + "@spider-cloud/spider-client": "^0.0.21", + "@supabase/supabase-js": "^2.45.0", + "@tensorflow-models/universal-sentence-encoder": "*", + "@tensorflow/tfjs-converter": "*", + "@tensorflow/tfjs-core": "*", + "@upstash/ratelimit": "^1.1.3", + "@upstash/redis": "^1.20.6", + "@upstash/vector": "^1.1.1", + "@vercel/kv": "^0.2.3", + "@vercel/postgres": "^0.5.0", + "@writerai/writer-sdk": "^0.40.2", + "@xata.io/client": "^0.28.0", + "@xenova/transformers": "^2.17.2", + "@zilliz/milvus2-sdk-node": ">=2.3.5", + "apify-client": "^2.7.1", + "assemblyai": "^4.6.0", + "better-sqlite3": ">=9.4.0 <12.0.0", + "cassandra-driver": "^4.7.2", + "cborg": "^4.1.1", + "cheerio": "^1.0.0-rc.12", + "chromadb": "*", + "closevector-common": "0.1.3", + "closevector-node": "0.1.6", + "closevector-web": "0.1.6", + "cohere-ai": "*", + "convex": "^1.3.1", + "couchbase": "^4.3.0", + "crypto-js": "^4.2.0", + "d3-dsv": "^2.0.0", + "discord.js": "^14.14.1", + "dria": "^0.0.3", + "duck-duck-scrape": "^2.2.5", + "epub2": "^3.0.1", + "faiss-node": "^0.5.1", + "firebase-admin": "^11.9.0 || ^12.0.0", + "google-auth-library": "*", + "googleapis": "*", + "hnswlib-node": "^3.0.0", + "html-to-text": "^9.0.5", + "ignore": "^5.2.0", + "interface-datastore": "^8.2.11", + "ioredis": "^5.3.2", + "it-all": "^3.0.4", + "jsdom": "*", + "jsonwebtoken": "^9.0.2", + "llmonitor": "^0.5.9", + "lodash": "^4.17.21", + "lunary": "^0.7.10", + "mammoth": "^1.6.0", + "mongodb": ">=5.2.0", + "mysql2": "^3.9.8", + "neo4j-driver": "*", + "node-llama-cpp": "*", + "notion-to-md": "^3.1.0", + "officeparser": "^4.0.4", + "pdf-parse": "1.1.1", + "pg": "^8.11.0", + "pg-copy-streams": "^6.0.5", + "pickleparser": "^0.2.1", + "playwright": "^1.32.1", + "portkey-ai": "^0.1.11", + "puppeteer": "*", + "redis": "*", + "replicate": "^0.29.4", + "sonix-speech-recognition": "^2.1.1", + "srt-parser-2": "^1.2.3", + "typeorm": "^0.3.20", + "typesense": "^1.5.3", + "usearch": "^1.1.1", + "vectordb": "^0.1.4", + "voy-search": "0.6.2", + "weaviate-ts-client": "*", + "web-auth-library": "^1.0.3", + "ws": "^8.14.2", + "youtube-transcript": "^1.0.6", + "youtubei.js": "^9.1.0" + }, + "peerDependenciesMeta": { + "@aws-crypto/sha256-js": { + "optional": true + }, + "@aws-sdk/client-bedrock-agent-runtime": { + "optional": true + }, + "@aws-sdk/client-bedrock-runtime": { + "optional": true + }, + "@aws-sdk/client-dynamodb": { + "optional": true + }, + "@aws-sdk/client-kendra": { + "optional": true + }, + "@aws-sdk/client-lambda": { + "optional": true + }, + "@aws-sdk/client-s3": { + "optional": true + }, + "@aws-sdk/client-sagemaker-runtime": { + "optional": true + }, + "@aws-sdk/client-sfn": { + "optional": true + }, + "@aws-sdk/credential-provider-node": { + "optional": true + }, + "@azure/search-documents": { + "optional": true + }, + "@azure/storage-blob": { + "optional": true + }, + "@browserbasehq/sdk": { + "optional": true + }, + "@clickhouse/client": { + "optional": true + }, + "@cloudflare/ai": { + "optional": true + }, + "@datastax/astra-db-ts": { + "optional": true + }, + "@elastic/elasticsearch": { + "optional": true + }, + "@getmetal/metal-sdk": { + "optional": true + }, + "@getzep/zep-cloud": { + "optional": true + }, + "@getzep/zep-js": { + "optional": true + }, + "@gomomento/sdk": { + "optional": true + }, + "@gomomento/sdk-core": { + "optional": true + }, + "@google-ai/generativelanguage": { + "optional": true + }, + "@google-cloud/storage": { + "optional": true + }, + "@gradientai/nodejs-sdk": { + "optional": true + }, + "@huggingface/inference": { + "optional": true + }, + "@langchain/langgraph": { + "optional": true + }, + "@layerup/layerup-security": { + "optional": true + }, + "@mendable/firecrawl-js": { + "optional": true + }, + "@mlc-ai/web-llm": { + "optional": true + }, + "@mozilla/readability": { + "optional": true + }, + "@neondatabase/serverless": { + "optional": true + }, + "@notionhq/client": { + "optional": true + }, + "@opensearch-project/opensearch": { + "optional": true + }, + "@pinecone-database/pinecone": { + "optional": true + }, + "@planetscale/database": { + "optional": true + }, + "@premai/prem-sdk": { + "optional": true + }, + "@qdrant/js-client-rest": { + "optional": true + }, + "@raycast/api": { + "optional": true + }, + "@rockset/client": { + "optional": true + }, + "@smithy/eventstream-codec": { + "optional": true + }, + "@smithy/protocol-http": { + "optional": true + }, + "@smithy/signature-v4": { + "optional": true + }, + "@smithy/util-utf8": { + "optional": true + }, + "@spider-cloud/spider-client": { + "optional": true + }, + "@supabase/supabase-js": { + "optional": true + }, + "@tensorflow-models/universal-sentence-encoder": { + "optional": true + }, + "@tensorflow/tfjs-converter": { + "optional": true + }, + "@tensorflow/tfjs-core": { + "optional": true + }, + "@upstash/ratelimit": { + "optional": true + }, + "@upstash/redis": { + "optional": true + }, + "@upstash/vector": { + "optional": true + }, + "@vercel/kv": { + "optional": true + }, + "@vercel/postgres": { + "optional": true + }, + "@writerai/writer-sdk": { + "optional": true + }, + "@xata.io/client": { + "optional": true + }, + "@xenova/transformers": { + "optional": true + }, + "@zilliz/milvus2-sdk-node": { + "optional": true + }, + "apify-client": { + "optional": true + }, + "assemblyai": { + "optional": true + }, + "better-sqlite3": { + "optional": true + }, + "cassandra-driver": { + "optional": true + }, + "cborg": { + "optional": true + }, + "cheerio": { + "optional": true + }, + "chromadb": { + "optional": true + }, + "closevector-common": { + "optional": true + }, + "closevector-node": { + "optional": true + }, + "closevector-web": { + "optional": true + }, + "cohere-ai": { + "optional": true + }, + "convex": { + "optional": true + }, + "couchbase": { + "optional": true + }, + "crypto-js": { + "optional": true + }, + "d3-dsv": { + "optional": true + }, + "discord.js": { + "optional": true + }, + "dria": { + "optional": true + }, + "duck-duck-scrape": { + "optional": true + }, + "epub2": { + "optional": true + }, + "faiss-node": { + "optional": true + }, + "firebase-admin": { + "optional": true + }, + "google-auth-library": { + "optional": true + }, + "googleapis": { + "optional": true + }, + "hnswlib-node": { + "optional": true + }, + "html-to-text": { + "optional": true + }, + "ignore": { + "optional": true + }, + "interface-datastore": { + "optional": true + }, + "ioredis": { + "optional": true + }, + "it-all": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "jsonwebtoken": { + "optional": true + }, + "llmonitor": { + "optional": true + }, + "lodash": { + "optional": true + }, + "lunary": { + "optional": true + }, + "mammoth": { + "optional": true + }, + "mongodb": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "neo4j-driver": { + "optional": true + }, + "node-llama-cpp": { + "optional": true + }, + "notion-to-md": { + "optional": true + }, + "officeparser": { + "optional": true + }, + "pdf-parse": { + "optional": true + }, + "pg": { + "optional": true + }, + "pg-copy-streams": { + "optional": true + }, + "pickleparser": { + "optional": true + }, + "playwright": { + "optional": true + }, + "portkey-ai": { + "optional": true + }, + "puppeteer": { + "optional": true + }, + "redis": { + "optional": true + }, + "replicate": { + "optional": true + }, + "sonix-speech-recognition": { + "optional": true + }, + "srt-parser-2": { + "optional": true + }, + "typeorm": { + "optional": true + }, + "typesense": { + "optional": true + }, + "usearch": { + "optional": true + }, + "vectordb": { + "optional": true + }, + "voy-search": { + "optional": true + }, + "weaviate-ts-client": { + "optional": true + }, + "web-auth-library": { + "optional": true + }, + "ws": { + "optional": true + }, + "youtube-transcript": { + "optional": true + }, + "youtubei.js": { + "optional": true + } + } + }, + "node_modules/@langchain/community/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@langchain/community/node_modules/langsmith": { + "version": "0.1.48", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.48.tgz", + "integrity": "sha512-lh98dZeShVPG1VzENpbjFWiburyYpChsO7nehGwxuqQ5/E1BBFKpsDCxLTcgXYpgWFPJxRvMqq7bLeq/txjakw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/uuid": "^10.0.0", + "commander": "^10.0.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "@langchain/core": "*", + "langchain": "*", + "openai": "*" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + }, + "langchain": { + "optional": true + }, + "openai": { + "optional": true + } + } + }, + "node_modules/@langchain/core": { + "version": "0.2.31", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.2.31.tgz", + "integrity": "sha512-qGfeaACST7dvovgHItzuag9fEBGK7IjCE9vRuTu/y8/WYGJi28WPD/AwWxnu7YdW1vZSuIXO6ZA76t2G9B/oKg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": "^0.1.43", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^10.0.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/core/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@langchain/core/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/core/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@langchain/core/node_modules/langsmith": { + "version": "0.1.48", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.48.tgz", + "integrity": "sha512-lh98dZeShVPG1VzENpbjFWiburyYpChsO7nehGwxuqQ5/E1BBFKpsDCxLTcgXYpgWFPJxRvMqq7bLeq/txjakw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/uuid": "^10.0.0", + "commander": "^10.0.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "@langchain/core": "*", + "langchain": "*", + "openai": "*" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + }, + "langchain": { + "optional": true + }, + "openai": { + "optional": true + } + } + }, + "node_modules/@langchain/ollama": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@langchain/ollama/-/ollama-0.0.4.tgz", + "integrity": "sha512-laYaYFZsbu0Mjhm40CypXPV5gYvQjd16oBEoGnFFjmCUgCCu6gjETg3tgvgifo7w+dhmNPLUyqnjXA/BpWEH2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@langchain/core": ">=0.2.21 <0.3.0", + "ollama": "^0.5.6", + "uuid": "^10.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/openai": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.2.8.tgz", + "integrity": "sha512-p5fxEAKuR8UV9jWIxkZ6AY/vAPSYxJI0Pf/UM4T3FKk/dn99G/mAEDLhfI4pBf7B8o8TudSVyBW2hRjZqlQu7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@langchain/core": ">=0.2.26 <0.3.0", + "js-tiktoken": "^1.0.12", + "openai": "^4.55.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/textsplitters": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@langchain/textsplitters/-/textsplitters-0.0.3.tgz", + "integrity": "sha512-cXWgKE3sdWLSqAa8ykbCcUsUF1Kyr5J3HOWYGuobhPEycXW4WI++d5DhzdpL238mzoEXTi90VqfSCra37l5YqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@langchain/core": ">0.2.0 <0.3.0", + "js-tiktoken": "^1.0.12" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@loaders.gl/core": { "version": "3.4.15", "resolved": "https://registry.npmjs.org/@loaders.gl/core/-/core-3.4.15.tgz", @@ -2401,6 +3121,18 @@ "dev": true, "license": "MIT" }, + "node_modules/@mozilla/readability": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.5.0.tgz", + "integrity": "sha512-Z+CZ3QaosfFaTqvhQsIktyGrjFjSC0Fa4EMph4mqKnWhmyoGICsV/8QK+8HpXut6zV7zwfWwqDmEjtk1Qf6EgQ==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "peer": true, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@nevware21/ts-async": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/@nevware21/ts-async/-/ts-async-0.5.2.tgz", @@ -2506,6 +3238,55 @@ "node": ">=14.5.0" } }, + "node_modules/@observablehq/stdlib/node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/@observablehq/stdlib/node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "dev": true, + "license": "ISC", + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@observablehq/stdlib/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/@octokit/auth-token": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-5.1.1.tgz", @@ -2906,15 +3687,26 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "20.16.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.1.tgz", - "integrity": "sha512-zJDo7wEadFtSyNz5QITDfRcrhqDvQI1xQNQ0VoizPjM/dVAODqqIUWbJPkvsxmTI0MYRGRikcdjMPhOssnPejQ==", + "version": "20.16.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.5.tgz", + "integrity": "sha512-VwYCweNo3ERajwy0IUlqqcyZ8/A7Zwa9ZP3MnENWcB11AejO+tLy3pu850goUW2FC/IJMdZUfKpX/yxL1gymCA==", "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.19.2" } }, + "node_modules/@types/node-fetch": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", + "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, "node_modules/@types/normalize-package-data": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", @@ -2936,6 +3728,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/qs": { + "version": "6.9.15", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.15.tgz", + "integrity": "sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/react": { "version": "17.0.80", "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.80.tgz", @@ -2958,6 +3757,13 @@ "@types/react": "^17" } }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/scheduler": { "version": "0.16.8", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz", @@ -2972,6 +3778,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/vscode": { "version": "1.92.0", "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.92.0.tgz", @@ -2987,17 +3800,17 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.2.0.tgz", - "integrity": "sha512-02tJIs655em7fvt9gps/+4k4OsKULYGtLBPJfOsmOq1+3cdClYiF0+d6mHu6qDnTcg88wJBkcPLpQhq7FyDz0A==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.4.0.tgz", + "integrity": "sha512-rg8LGdv7ri3oAlenMACk9e+AR4wUV0yrrG+XKsGKOK0EVgeEDqurkXMPILG2836fW4ibokTB5v4b6Z9+GYQDEw==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "8.2.0", - "@typescript-eslint/type-utils": "8.2.0", - "@typescript-eslint/utils": "8.2.0", - "@typescript-eslint/visitor-keys": "8.2.0", + "@typescript-eslint/scope-manager": "8.4.0", + "@typescript-eslint/type-utils": "8.4.0", + "@typescript-eslint/utils": "8.4.0", + "@typescript-eslint/visitor-keys": "8.4.0", "graphemer": "^1.4.0", "ignore": "^5.3.1", "natural-compare": "^1.4.0", @@ -3021,16 +3834,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.2.0.tgz", - "integrity": "sha512-j3Di+o0lHgPrb7FxL3fdEy6LJ/j2NE8u+AP/5cQ9SKb+JLH6V6UHDqJ+e0hXBkHP1wn1YDFjYCS9LBQsZDlDEg==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.4.0.tgz", + "integrity": "sha512-NHgWmKSgJk5K9N16GIhQ4jSobBoJwrmURaLErad0qlLjrpP5bECYg+wxVTGlGZmJbU03jj/dfnb6V9bw+5icsA==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "@typescript-eslint/scope-manager": "8.2.0", - "@typescript-eslint/types": "8.2.0", - "@typescript-eslint/typescript-estree": "8.2.0", - "@typescript-eslint/visitor-keys": "8.2.0", + "@typescript-eslint/scope-manager": "8.4.0", + "@typescript-eslint/types": "8.4.0", + "@typescript-eslint/typescript-estree": "8.4.0", + "@typescript-eslint/visitor-keys": "8.4.0", "debug": "^4.3.4" }, "engines": { @@ -3050,14 +3863,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.2.0.tgz", - "integrity": "sha512-OFn80B38yD6WwpoHU2Tz/fTz7CgFqInllBoC3WP+/jLbTb4gGPTy9HBSTsbDWkMdN55XlVU0mMDYAtgvlUspGw==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.4.0.tgz", + "integrity": "sha512-n2jFxLeY0JmKfUqy3P70rs6vdoPjHK8P/w+zJcV3fk0b0BwRXC/zxRTEnAsgYT7MwdQDt/ZEbtdzdVC+hcpF0A==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.2.0", - "@typescript-eslint/visitor-keys": "8.2.0" + "@typescript-eslint/types": "8.4.0", + "@typescript-eslint/visitor-keys": "8.4.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3068,14 +3881,14 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.2.0.tgz", - "integrity": "sha512-g1CfXGFMQdT5S+0PSO0fvGXUaiSkl73U1n9LTK5aRAFnPlJ8dLKkXr4AaLFvPedW8lVDoMgLLE3JN98ZZfsj0w==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.4.0.tgz", + "integrity": "sha512-pu2PAmNrl9KX6TtirVOrbLPLwDmASpZhK/XU7WvoKoCUkdtq9zF7qQ7gna0GBZFN0hci0vHaSusiL2WpsQk37A==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/typescript-estree": "8.2.0", - "@typescript-eslint/utils": "8.2.0", + "@typescript-eslint/typescript-estree": "8.4.0", + "@typescript-eslint/utils": "8.4.0", "debug": "^4.3.4", "ts-api-utils": "^1.3.0" }, @@ -3093,9 +3906,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.2.0.tgz", - "integrity": "sha512-6a9QSK396YqmiBKPkJtxsgZZZVjYQ6wQ/TlI0C65z7vInaETuC6HAHD98AGLC8DyIPqHytvNuS8bBVvNLKyqvQ==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.4.0.tgz", + "integrity": "sha512-T1RB3KQdskh9t3v/qv7niK6P8yvn7ja1mS7QK7XfRVL6wtZ8/mFs/FHf4fKvTA0rKnqnYxl/uHFNbnEt0phgbw==", "dev": true, "license": "MIT", "engines": { @@ -3107,16 +3920,16 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.2.0.tgz", - "integrity": "sha512-kiG4EDUT4dImplOsbh47B1QnNmXSoUqOjWDvCJw/o8LgfD0yr7k2uy54D5Wm0j4t71Ge1NkynGhpWdS0dEIAUA==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.4.0.tgz", + "integrity": "sha512-kJ2OIP4dQw5gdI4uXsaxUZHRwWAGpREJ9Zq6D5L0BweyOrWsL6Sz0YcAZGWhvKnH7fm1J5YFE1JrQL0c9dd53A==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "@typescript-eslint/types": "8.2.0", - "@typescript-eslint/visitor-keys": "8.2.0", + "@typescript-eslint/types": "8.4.0", + "@typescript-eslint/visitor-keys": "8.4.0", "debug": "^4.3.4", - "globby": "^11.1.0", + "fast-glob": "^3.3.2", "is-glob": "^4.0.3", "minimatch": "^9.0.4", "semver": "^7.6.0", @@ -3136,16 +3949,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.2.0.tgz", - "integrity": "sha512-O46eaYKDlV3TvAVDNcoDzd5N550ckSe8G4phko++OCSC1dYIb9LTc3HDGYdWqWIAT5qDUKphO6sd9RrpIJJPfg==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.4.0.tgz", + "integrity": "sha512-swULW8n1IKLjRAgciCkTCafyTHHfwVQFt8DovmaF69sKbOxTSFMmIZaSHjqO9i/RV0wIblaawhzvtva8Nmm7lQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "8.2.0", - "@typescript-eslint/types": "8.2.0", - "@typescript-eslint/typescript-estree": "8.2.0" + "@typescript-eslint/scope-manager": "8.4.0", + "@typescript-eslint/types": "8.4.0", + "@typescript-eslint/typescript-estree": "8.4.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3159,13 +3972,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.2.0.tgz", - "integrity": "sha512-sbgsPMW9yLvS7IhCi8IpuK1oBmtbWUNP+hBdwl/I9nzqVsszGnNGti5r9dUtF5RLivHUFFIdRvLiTsPhzSyJ3Q==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.4.0.tgz", + "integrity": "sha512-zTQD6WLNTre1hj5wp09nBIDiOc2U5r/qmzo7wxPn4ZgAjHql09EofqhF9WF+fZHzL5aCyaIpPcT2hyxl73kr9A==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.2.0", + "@typescript-eslint/types": "8.4.0", "eslint-visitor-keys": "^3.4.3" }, "engines": { @@ -3184,22 +3997,22 @@ "license": "ISC" }, "node_modules/@vscode/debugadapter": { - "version": "1.66.0", - "resolved": "https://registry.npmjs.org/@vscode/debugadapter/-/debugadapter-1.66.0.tgz", - "integrity": "sha512-U/m5l6igHtQ8rSMSKW9oWeco9ySPqGYjqW9NECGPGWZ/xnoYicpqUoXhGx3xUNsafrinzWvUWrSUL/Cdgj2V+w==", + "version": "1.67.0", + "resolved": "https://registry.npmjs.org/@vscode/debugadapter/-/debugadapter-1.67.0.tgz", + "integrity": "sha512-AzCiKbRdOheHFaTJ2BVdfGuREIOhNEtMUVDv4lJqIPCfZ/60tXELvVSOFGG+6hNMiMaSrAcXP9sPSlZLZEsmLA==", "dev": true, "license": "MIT", "dependencies": { - "@vscode/debugprotocol": "1.66.0" + "@vscode/debugprotocol": "1.67.0" }, "engines": { "node": ">=14" } }, "node_modules/@vscode/debugprotocol": { - "version": "1.66.0", - "resolved": "https://registry.npmjs.org/@vscode/debugprotocol/-/debugprotocol-1.66.0.tgz", - "integrity": "sha512-VGcRBLNL8QwHzwerSWOb60fy1FO7bdseZv6OkTS4opoP3xeyDX58i4/wAwakL2Y4P9NafN4VGrvlXSWIratmWA==", + "version": "1.67.0", + "resolved": "https://registry.npmjs.org/@vscode/debugprotocol/-/debugprotocol-1.67.0.tgz", + "integrity": "sha512-vTn5JwZ+LQy2QqT/wUD8Rlrb+7eLo5fsiKIxD5i0BZIuvdRbxTTfGU7+47PsorMrfBzozngIrocKCKS3OVnYyw==", "dev": true, "license": "MIT" }, @@ -3219,15 +4032,15 @@ } }, "node_modules/@vscode/prompt-tsx": { - "version": "0.2.6-alpha", - "resolved": "https://registry.npmjs.org/@vscode/prompt-tsx/-/prompt-tsx-0.2.6-alpha.tgz", - "integrity": "sha512-eijaM5kMFez6rHgSIivBvBBiYA9n0IeHXXinML74JqQV/lgCqoGNQ9Hlzh38P8KJLJ8aV8f14q6vPRIXbJfteg==", + "version": "0.2.8-alpha", + "resolved": "https://registry.npmjs.org/@vscode/prompt-tsx/-/prompt-tsx-0.2.8-alpha.tgz", + "integrity": "sha512-XsUybYTLofhKRzywACt0aGObnYnBABkRMmj4mMyYa93lH15MXwRxEkqY4ALIFKjhchqGamOoy8I1RcS++tgaDw==", "license": "SEE LICENSE IN LICENSE" }, "node_modules/@vscode/vsce": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-3.0.0.tgz", - "integrity": "sha512-UKYcC7fcSw6AUK6nlfm8A8WSOA41H3DRAwafCMp9CQpg3K9COAQKkrgQ+dKPhwkFTP7SPZNEiulDpPLwvr5QQQ==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-3.1.0.tgz", + "integrity": "sha512-fwdfp1Ol+bZtlSGkpcd/nztfo6+SVsTOMWjZ/+a88lVtUn7gXNbSu7dbniecl5mz4vINl+oaVDVtVdGbJDApmw==", "dev": true, "license": "MIT", "dependencies": { @@ -3243,7 +4056,7 @@ "hosted-git-info": "^4.0.2", "jsonc-parser": "^3.2.0", "leven": "^3.1.0", - "markdown-it": "^12.3.2", + "markdown-it": "^14.1.0", "mime": "^1.3.4", "minimatch": "^3.0.3", "parse-semver": "^1.1.1", @@ -3552,9 +4365,9 @@ "license": "MIT" }, "node_modules/adm-zip": { - "version": "0.5.15", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.15.tgz", - "integrity": "sha512-jYPWSeOA8EFoZnucrKCNihqBjoEGQSU4HKgHYQgKNEQ0pQF9a/DYuo/+fAxY76k4qe75LUlLWpAM1QWcBMTOKw==", + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", "dev": true, "license": "MIT", "engines": { @@ -3568,10 +4381,23 @@ "dev": true, "license": "MIT", "dependencies": { - "debug": "^4.3.4" + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/agentkeepalive": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "dev": true, + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" }, "engines": { - "node": ">= 14" + "node": ">= 8.0.0" } }, "node_modules/ajv": { @@ -3754,16 +4580,6 @@ "dev": true, "license": "MIT" }, - "node_modules/array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/arraybuffer.prototype.slice": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.3.tgz", @@ -3919,6 +4735,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -4834,6 +5662,30 @@ "dev": true, "license": "MIT" }, + "node_modules/cssstyle": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.0.1.tgz", + "integrity": "sha512-8ZYiJ3A/3OkDd093CBT/0UKDWry7ak4BdPTFP2+QEP7cmhouyq/Up709ASSj2cK02BbZiMgk7kYjZNS4QP5qrQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "rrweb-cssom": "^0.6.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/cssstyle/node_modules/rrweb-cssom": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz", + "integrity": "sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/csstype": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", @@ -4862,40 +5714,38 @@ } }, "node_modules/d3-dsv": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", - "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-2.0.0.tgz", + "integrity": "sha512-E+Pn8UJYx9mViuIUkoc93gJGGYut6mSDKy2+XaPwccwkRGlR+LO97L2VCCRjQivTwLHkSnAJG7yo00BWY6QM+w==", "dev": true, - "license": "ISC", + "license": "BSD-3-Clause", + "optional": true, + "peer": true, "dependencies": { - "commander": "7", - "iconv-lite": "0.6", + "commander": "2", + "iconv-lite": "0.4", "rw": "1" }, "bin": { - "csv2json": "bin/dsv2json.js", - "csv2tsv": "bin/dsv2dsv.js", - "dsv2dsv": "bin/dsv2dsv.js", - "dsv2json": "bin/dsv2json.js", - "json2csv": "bin/json2dsv.js", - "json2dsv": "bin/json2dsv.js", - "json2tsv": "bin/json2dsv.js", - "tsv2csv": "bin/dsv2dsv.js", - "tsv2json": "bin/dsv2json.js" - }, - "engines": { - "node": ">=12" + "csv2json": "bin/dsv2json", + "csv2tsv": "bin/dsv2dsv", + "dsv2dsv": "bin/dsv2dsv", + "dsv2json": "bin/dsv2json", + "json2csv": "bin/json2dsv", + "json2dsv": "bin/json2dsv", + "json2tsv": "bin/json2dsv", + "tsv2csv": "bin/dsv2dsv", + "tsv2json": "bin/dsv2json" } }, "node_modules/d3-dsv/node_modules/commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true, "license": "MIT", - "engines": { - "node": ">= 10" - } + "optional": true, + "peer": true }, "node_modules/d3-require": { "version": "1.3.0", @@ -4924,6 +5774,65 @@ "node": ">= 12" } }, + "node_modules/data-urls": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", + "integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/data-urls/node_modules/tr46": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.0.0.tgz", + "integrity": "sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/data-urls/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dev": true, + "license": "BSD-2-Clause", + "optional": true, + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/data-urls/node_modules/whatwg-url": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.0.0.tgz", + "integrity": "sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tr46": "^5.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/data-view-buffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.1.tgz", @@ -5049,6 +5958,15 @@ "node": ">=0.10.0" } }, + "node_modules/decimal.js": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", + "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/decompress-response": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", @@ -5190,19 +6108,6 @@ "node": ">=0.3.1" } }, - "node_modules/dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-type": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -6008,6 +6913,13 @@ "node": ">=6" } }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, + "license": "MIT" + }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -6058,6 +6970,40 @@ "node": ">=6" } }, + "node_modules/expr-eval": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/expr-eval/-/expr-eval-2.0.2.tgz", + "integrity": "sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg==", + "dev": true, + "license": "MIT" + }, + "node_modules/faiss-node": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/faiss-node/-/faiss-node-0.5.1.tgz", + "integrity": "sha512-zD8wobJn8C6OLWo68Unho+Ih8l6nSRB2w3Amj01a+xc4bsEvd2mBDLklAn7VocA9XO3WDvQL/bLpi5flkCn/XQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "bindings": "^1.5.0", + "node-addon-api": "^6.0.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/faiss-node/node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/fast-csv": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", @@ -6195,6 +7141,15 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -6316,6 +7271,37 @@ "node": ">= 6" } }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "dev": true, + "license": "MIT" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/formdata-node/node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -6756,27 +7742,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "license": "MIT", - "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -6963,6 +7928,21 @@ "node": ">=10" } }, + "node_modules/html-encoding-sniffer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", + "integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "whatwg-encoding": "^3.1.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/html-to-text": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-9.0.5.tgz", @@ -7026,14 +8006,26 @@ "node": ">= 14" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" + "safer-buffer": ">= 2.1.2 < 3" }, "engines": { "node": ">=0.10.0" @@ -7457,6 +8449,15 @@ "node": ">=0.10.0" } }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/is-regex": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", @@ -7616,38 +8617,149 @@ "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.0.1.tgz", "integrity": "sha512-cub8rahkh0Q/bw1+GxP7aeSe29hHHn2V4m29nnDlvCdlgU+3UGxkZp7Z53jLUdpX3jdTO0nJZUDl3xvbWc2Xog==", "dev": true, - "license": "BlueOak-1.0.0", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.14.tgz", + "integrity": "sha512-Pk3l3WOgM9joguZY2k52+jH82RtABRgB5RdGFZNUGbOKGMVlNmafcPA3b0ITcCZPu1L9UclP1tne6aw7ZI4Myg==", + "dev": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsdom": { + "version": "25.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-25.0.0.tgz", + "integrity": "sha512-OhoFVT59T7aEq75TVw9xxEfkXgacpqAhQaYgP9y/fDqWQCMB/b1H66RfmPm/MaeaAIU9nDwMOVTlPN51+ao6CQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "cssstyle": "^4.0.1", + "data-urls": "^5.0.0", + "decimal.js": "^10.4.3", + "form-data": "^4.0.0", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.5", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.12", + "parse5": "^7.1.2", + "rrweb-cssom": "^0.7.1", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^4.1.4", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "canvas": "^2.11.2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/jsdom/node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "dev": true, + "license": "ISC", + "optional": true, + "peer": true, "dependencies": { - "@isaacs/cliui": "^8.0.2" + "xmlchars": "^2.2.0" }, "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "node": ">=v12.22.7" + } + }, + "node_modules/jsdom/node_modules/tr46": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.0.0.tgz", + "integrity": "sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "punycode": "^2.3.1" }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" + "engines": { + "node": ">=18" } }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "node_modules/jsdom/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", "dev": true, - "license": "MIT" + "license": "BSD-2-Clause", + "optional": true, + "peer": true, + "engines": { + "node": ">=12" + } }, - "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "node_modules/jsdom/node_modules/whatwg-url": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.0.0.tgz", + "integrity": "sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { - "argparse": "^2.0.1" + "tr46": "^5.0.0", + "webidl-conversions": "^7.0.0" }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "engines": { + "node": ">=18" } }, "node_modules/json-buffer": { @@ -7709,6 +8821,16 @@ ], "license": "MIT" }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/JSONStream": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", @@ -7883,6 +9005,312 @@ "node": ">=0.10.0" } }, + "node_modules/langchain": { + "version": "0.2.18", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.2.18.tgz", + "integrity": "sha512-7+5Y2FsdjlUBKJf1N+MyHn1PrGZQ9m87l3du7IeH4RiQp4j8Bz7mGV+Bjhr8FEXFjo85zLrbnpsN87pZCstubA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@langchain/core": ">=0.2.21 <0.3.0", + "@langchain/openai": ">=0.1.0 <0.3.0", + "@langchain/textsplitters": "~0.0.0", + "binary-extensions": "^2.2.0", + "js-tiktoken": "^1.0.12", + "js-yaml": "^4.1.0", + "jsonpointer": "^5.0.1", + "langsmith": "~0.1.40", + "openapi-types": "^12.1.3", + "p-retry": "4", + "uuid": "^10.0.0", + "yaml": "^2.2.1", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "*", + "@aws-sdk/client-sagemaker-runtime": "*", + "@aws-sdk/client-sfn": "*", + "@aws-sdk/credential-provider-node": "*", + "@azure/storage-blob": "*", + "@browserbasehq/sdk": "*", + "@gomomento/sdk": "*", + "@gomomento/sdk-core": "*", + "@gomomento/sdk-web": "^1.51.1", + "@langchain/anthropic": "*", + "@langchain/aws": "*", + "@langchain/cohere": "*", + "@langchain/community": "*", + "@langchain/google-genai": "*", + "@langchain/google-vertexai": "*", + "@langchain/groq": "*", + "@langchain/mistralai": "*", + "@langchain/ollama": "*", + "@mendable/firecrawl-js": "*", + "@notionhq/client": "*", + "@pinecone-database/pinecone": "*", + "@supabase/supabase-js": "*", + "@vercel/kv": "*", + "@xata.io/client": "*", + "apify-client": "*", + "assemblyai": "*", + "axios": "*", + "cheerio": "*", + "chromadb": "*", + "convex": "*", + "couchbase": "*", + "d3-dsv": "*", + "epub2": "*", + "fast-xml-parser": "*", + "handlebars": "^4.7.8", + "html-to-text": "*", + "ignore": "*", + "ioredis": "*", + "jsdom": "*", + "mammoth": "*", + "mongodb": "*", + "node-llama-cpp": "*", + "notion-to-md": "*", + "officeparser": "*", + "pdf-parse": "*", + "peggy": "^3.0.2", + "playwright": "*", + "puppeteer": "*", + "pyodide": ">=0.24.1 <0.27.0", + "redis": "*", + "sonix-speech-recognition": "*", + "srt-parser-2": "*", + "typeorm": "*", + "weaviate-ts-client": "*", + "web-auth-library": "*", + "ws": "*", + "youtube-transcript": "*", + "youtubei.js": "*" + }, + "peerDependenciesMeta": { + "@aws-sdk/client-s3": { + "optional": true + }, + "@aws-sdk/client-sagemaker-runtime": { + "optional": true + }, + "@aws-sdk/client-sfn": { + "optional": true + }, + "@aws-sdk/credential-provider-node": { + "optional": true + }, + "@azure/storage-blob": { + "optional": true + }, + "@browserbasehq/sdk": { + "optional": true + }, + "@gomomento/sdk": { + "optional": true + }, + "@gomomento/sdk-core": { + "optional": true + }, + "@gomomento/sdk-web": { + "optional": true + }, + "@langchain/anthropic": { + "optional": true + }, + "@langchain/aws": { + "optional": true + }, + "@langchain/cohere": { + "optional": true + }, + "@langchain/community": { + "optional": true + }, + "@langchain/google-genai": { + "optional": true + }, + "@langchain/google-vertexai": { + "optional": true + }, + "@langchain/groq": { + "optional": true + }, + "@langchain/mistralai": { + "optional": true + }, + "@langchain/ollama": { + "optional": true + }, + "@mendable/firecrawl-js": { + "optional": true + }, + "@notionhq/client": { + "optional": true + }, + "@pinecone-database/pinecone": { + "optional": true + }, + "@supabase/supabase-js": { + "optional": true + }, + "@vercel/kv": { + "optional": true + }, + "@xata.io/client": { + "optional": true + }, + "apify-client": { + "optional": true + }, + "assemblyai": { + "optional": true + }, + "axios": { + "optional": true + }, + "cheerio": { + "optional": true + }, + "chromadb": { + "optional": true + }, + "convex": { + "optional": true + }, + "couchbase": { + "optional": true + }, + "d3-dsv": { + "optional": true + }, + "epub2": { + "optional": true + }, + "faiss-node": { + "optional": true + }, + "fast-xml-parser": { + "optional": true + }, + "handlebars": { + "optional": true + }, + "html-to-text": { + "optional": true + }, + "ignore": { + "optional": true + }, + "ioredis": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "mammoth": { + "optional": true + }, + "mongodb": { + "optional": true + }, + "node-llama-cpp": { + "optional": true + }, + "notion-to-md": { + "optional": true + }, + "officeparser": { + "optional": true + }, + "pdf-parse": { + "optional": true + }, + "peggy": { + "optional": true + }, + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + }, + "pyodide": { + "optional": true + }, + "redis": { + "optional": true + }, + "sonix-speech-recognition": { + "optional": true + }, + "srt-parser-2": { + "optional": true + }, + "typeorm": { + "optional": true + }, + "weaviate-ts-client": { + "optional": true + }, + "web-auth-library": { + "optional": true + }, + "ws": { + "optional": true + }, + "youtube-transcript": { + "optional": true + }, + "youtubei.js": { + "optional": true + } + } + }, + "node_modules/langchain/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/langchain/node_modules/langsmith": { + "version": "0.1.48", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.48.tgz", + "integrity": "sha512-lh98dZeShVPG1VzENpbjFWiburyYpChsO7nehGwxuqQ5/E1BBFKpsDCxLTcgXYpgWFPJxRvMqq7bLeq/txjakw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/uuid": "^10.0.0", + "commander": "^10.0.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "@langchain/core": "*", + "langchain": "*", + "openai": "*" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + }, + "langchain": { + "optional": true + }, + "openai": { + "optional": true + } + } + }, "node_modules/lazystream": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", @@ -7988,13 +9416,13 @@ "license": "MIT" }, "node_modules/linkify-it": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz", - "integrity": "sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", "dev": true, "license": "MIT", "dependencies": { - "uc.micro": "^1.0.1" + "uc.micro": "^2.0.0" } }, "node_modules/listenercount": { @@ -8333,30 +9761,21 @@ } }, "node_modules/markdown-it": { - "version": "12.3.2", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-12.3.2.tgz", - "integrity": "sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", + "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", "dev": true, "license": "MIT", "dependencies": { "argparse": "^2.0.1", - "entities": "~2.1.0", - "linkify-it": "^3.0.1", - "mdurl": "^1.0.1", - "uc.micro": "^1.0.5" + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" }, "bin": { - "markdown-it": "bin/markdown-it.js" - } - }, - "node_modules/markdown-it/node_modules/entities": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", - "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==", - "dev": true, - "license": "BSD-2-Clause", - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" + "markdown-it": "bin/markdown-it.mjs" } }, "node_modules/math.gl": { @@ -8370,9 +9789,9 @@ } }, "node_modules/mdurl": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", - "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", "dev": true, "license": "MIT" }, @@ -8610,9 +10029,9 @@ } }, "node_modules/micromatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", - "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, "license": "MIT", "dependencies": { @@ -8940,6 +10359,16 @@ "dev": true, "license": "MIT" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "dev": true, + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -9252,6 +10681,15 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, + "node_modules/nwsapi": { + "version": "2.2.12", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.12.tgz", + "integrity": "sha512-qXDmcVlZV4XRtKFzddidpfVP4oMSGhga+xdMc25mv8kaLUHtgzCDhUxkrN8exkGdTlLNaXj7CV3GtON7zuGZ+w==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -9321,6 +10759,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/ollama": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.8.tgz", + "integrity": "sha512-frBGdfSV34i7JybLZUeyCYDx0CMyDiG4On8xOK+cNRWM04HImhoWgIMpF4p7vTkQumadbSxOteR7SZyKqNmOXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-fetch": "^3.6.20" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -9330,24 +10778,77 @@ "wrappy": "1" } }, - "node_modules/open": { - "version": "8.4.2", - "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", - "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "node_modules/open": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-lazy-prop": "^2.0.0", + "is-docker": "^2.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/openai": { + "version": "4.57.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.57.0.tgz", + "integrity": "sha512-JnwBSIYqiZ3jYjB5f2in8hQ0PRA092c6m+/6dYB0MzK0BEbn+0dioxZsPLBm5idJbg9xzLNOiGVm2OSuhZ+BdQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "@types/qs": "^6.9.7", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "qs": "^6.10.3" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/openai/node_modules/@types/node": { + "version": "18.19.47", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.47.tgz", + "integrity": "sha512-1f7dB3BL/bpd9tnDJrrHb66Y+cVrhxSOTGorRNdHwYTUlTay3HuTDPKo9a/4vX9pMQkhYBcAbL4jQdNlhCFP9A==", "dev": true, "license": "MIT", "dependencies": { - "define-lazy-prop": "^2.0.0", - "is-docker": "^2.1.1", - "is-wsl": "^2.2.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "undici-types": "~5.26.4" } }, + "node_modules/openai/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", + "dev": true, + "license": "MIT" + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -9366,6 +10867,16 @@ "node": ">= 0.8.0" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -9398,6 +10909,50 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -9571,16 +11126,6 @@ "node": "20 || >=22" } }, - "node_modules/path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/pbf": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/pbf/-/pbf-3.3.0.tgz", @@ -9733,6 +11278,15 @@ "dev": true, "license": "MIT" }, + "node_modules/psl": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -9755,6 +11309,16 @@ "node": ">=6" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", @@ -9783,6 +11347,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -10114,6 +11687,15 @@ "node": ">=0.10.0" } }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/resolve": { "version": "1.22.8", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", @@ -10152,6 +11734,16 @@ "protocol-buffers-schema": "^3.3.1" } }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -10223,6 +11815,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rrweb-cssom": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.7.1.tgz", + "integrity": "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -10247,6 +11848,17 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/run-script-os": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/run-script-os/-/run-script-os-1.1.6.tgz", + "integrity": "sha512-ql6P2LzhBTTDfzKts+Qo4H94VUKpxKDFz6QxxwaUZN0mwvi7L3lpOI7BqPCq7lgDh3XLl0dpeXwfcVIitlrYrw==", + "dev": true, + "license": "MIT", + "bin": { + "run-os": "index.js", + "run-script-os": "index.js" + } + }, "node_modules/rw": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", @@ -10361,9 +11973,9 @@ } }, "node_modules/semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "dev": true, "license": "ISC", "bin": { @@ -10537,16 +12149,6 @@ "simple-concat": "^1.0.0" } }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -10905,6 +12507,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/tar-fs": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", @@ -11046,6 +12657,24 @@ "dev": true, "license": "MIT" }, + "node_modules/tough-cookie": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz", + "integrity": "sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==", + "dev": true, + "license": "BSD-3-Clause", + "optional": true, + "peer": true, + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -11096,9 +12725,9 @@ } }, "node_modules/tslib": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", - "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==", "dev": true, "license": "0BSD" }, @@ -11263,15 +12892,15 @@ } }, "node_modules/typescript-eslint": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.2.0.tgz", - "integrity": "sha512-DmnqaPcML0xYwUzgNbM1XaKXpEb7BShYf2P1tkUmmcl8hyeG7Pj08Er7R9bNy6AufabywzJcOybQAtnD/c9DGw==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.4.0.tgz", + "integrity": "sha512-67qoc3zQZe3CAkO0ua17+7aCLI0dU+sSQd1eKPGq06QE4rfQjstVXR6woHO5qQvGUa550NfGckT4tzh3b3c8Pw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.2.0", - "@typescript-eslint/parser": "8.2.0", - "@typescript-eslint/utils": "8.2.0" + "@typescript-eslint/eslint-plugin": "8.4.0", + "@typescript-eslint/parser": "8.4.0", + "@typescript-eslint/utils": "8.4.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -11287,9 +12916,9 @@ } }, "node_modules/uc.micro": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz", - "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", "dev": true, "license": "MIT" }, @@ -11357,6 +12986,18 @@ "dev": true, "license": "ISC" }, + "node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", @@ -11438,6 +13079,19 @@ "dev": true, "license": "MIT" }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/use-resize-observer": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/use-resize-observer/-/use-resize-observer-9.1.0.tgz", @@ -11452,6 +13106,42 @@ "react-dom": "16.8.0 - 18" } }, + "node_modules/usearch": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/usearch/-/usearch-1.3.0.tgz", + "integrity": "sha512-Wnesgqqvzj9jbqz48MDYRrR08WMbk73FKoYDPOeoa1H0jpQWfUltKv2xo+eywjoSL+IHUf1I9UjP17j+Z4OOMA==", + "dev": true, + "hasInstallScript": true, + "license": "Apache 2.0", + "optional": true, + "peer": true, + "dependencies": { + "@types/node": "^20.4.5", + "bindings": "~1.2.1", + "node-addon-api": "^3.0.0" + }, + "engines": { + "node": "~10 >=10.20 || >=12.17" + } + }, + "node_modules/usearch/node_modules/bindings": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "integrity": "sha512-u4cBQNepWxYA55FunZSM7wMi55yQaN0otnhhilNoWHq0MfOfJeQx0v0mRRpolGOExPjZcl6FtB0BB8Xkb88F0g==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, + "node_modules/usearch/node_modules/node-addon-api": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz", + "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/util": { "version": "0.12.5", "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", @@ -11509,6 +13199,21 @@ "pbf": "^3.2.1" } }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/web-streams-polyfill": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", @@ -11526,6 +13231,55 @@ "dev": true, "license": "BSD-2-Clause" }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/whatwg-fetch": { + "version": "3.6.20", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", + "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">=18" + } + }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", @@ -11729,6 +13483,42 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "peer": true, + "engines": { + "node": ">=18" + } + }, "node_modules/xml2js": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", @@ -11786,6 +13576,19 @@ "dev": true, "license": "ISC" }, + "node_modules/yaml": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.5.0.tgz", + "integrity": "sha512-2wWLbGbYDiSqqIKoPjar3MPgB94ErzCtrNE1FdqGuaO0pi2JGjmE8aW8TDZwzU7vuxcGRdL/4gPQwQ7hD5AMSw==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", @@ -11945,6 +13748,26 @@ "engines": { "node": ">= 10" } + }, + "node_modules/zod": { + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.23.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.23.2.tgz", + "integrity": "sha512-uSt90Gzc/tUfyNqxnjlfBs8W6WSGpNBv0rVsNxP/BVSMHMKGdthPYff4xtCHYloJGM0CFxFsb3NbC0eqPhfImw==", + "dev": true, + "license": "ISC", + "peerDependencies": { + "zod": "^3.23.3" + } } } } diff --git a/package.json b/package.json index f754baf..22061d2 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,12 @@ "browser": "./dist/web-extension", "scripts": { "clean": "rimraf --glob out lib* dist* types hpcc-systems.ecl.vsix", - "gen-grammar": "wsl -e ./scripts/grammar-generate.sh", + "gen-grammar": "run-script-os", + "gen-grammar:linux": "./scripts/grammar-generate.sh", + "gen-grammar:win32": "wsl -e ./scripts/grammar-generate.sh", + "gen-docs-vecdb": "run-script-os", + "gen-docs-vecdb:linux": "./dist-util/index-docs.mjs", + "gen-docs-vecdb:win32": "wsl -e node ./dist-util/index-docs.mjs", "gen-nls": "node ./lib-util/generate.js", "merge-nls": "node ./lib-util/merge.js", "gen-node-types": "tsc --project tsconfig.json --emitDeclarationOnly", @@ -24,11 +29,9 @@ "gen-webview-types": "tsc --project tsconfig.webview.json --emitDeclarationOnly", "gen-webview-watch": "npm run gen-webview-types -- -w", "gen-types": "run-p gen-node-types gen-webview-types", - "compile-util": "tsc --project ./tsconfig.util.json", - "compile-util-watch": "npm run compile-util -- -w", "build-ts": "node esbuild.mjs --production", - "build-ts-dev": "node esbuild.mjs", - "build-ts-watch": "node esbuild.mjs --watch", + "build-ts-dev": "node esbuild.mjs --development", + "build-ts-watch": "node esbuild.mjs --watch --development", "build": "run-p gen-types build-ts", "watch": "run-p gen-types-watch build-ts-watch", "lint": "eslint ./src", @@ -48,42 +51,48 @@ "update": "npx npm-check-updates -u -t minor" }, "dependencies": { - "@vscode/prompt-tsx": "0.2.6-alpha", + "@vscode/prompt-tsx": "0.2.8-alpha", "exceljs": "4.4.0", - "html-to-text": "^9.0.5" + "html-to-text": "9.0.5" }, "devDependencies": { - "@eslint/js": "9.9.0", - "@fluentui/react": "8.120.6", + "@eslint/js": "9.9.1", + "@fluentui/react": "8.120.7", "@hpcc-js/common": "2.72.0", "@hpcc-js/comms": "2.95.0", "@hpcc-js/ddl-shim": "2.21.0", "@hpcc-js/dgrid2": "2.4.0", - "@hpcc-js/esbuild-plugins": "1.1.0", - "@hpcc-js/loader": "2.106.4", + "@hpcc-js/esbuild-plugins": "1.1.2", + "@hpcc-js/loader": "2.106.7", "@hpcc-js/marshaller": "2.26.4", "@hpcc-js/observable-shim": "2.6.0", "@hpcc-js/observablehq-compiler": "1.3.0", "@hpcc-js/util": "2.52.0", + "@hpcc-js/wasm-expat": "1.3.0", + "@hpcc-js/wasm-llama": "1.0.0", + "@hpcc-js/wasm-zstd": "1.2.0", + "@langchain/community": "0.2.32", + "@langchain/core": "0.2.31", + "@langchain/ollama": "0.0.4", "@observablehq/inspector": "5.0.1", "@observablehq/parser": "6.1.0", "@observablehq/runtime": "5.9.9", "@observablehq/stdlib": "5.8.8", "@octokit/rest": "21.0.2", "@types/mocha": "10.0.7", - "@types/node": "20.16.1", + "@types/node": "20.16.5", "@types/react": "17.0.80", "@types/react-dom": "17.0.25", "@types/tmp": "0.2.6", "@types/vscode": "1.92.0", "@types/vscode-notebook-renderer": "1.72.3", - "@typescript-eslint/eslint-plugin": "8.2.0", - "@typescript-eslint/parser": "8.2.0", - "@vscode/debugadapter": "1.66.0", - "@vscode/debugprotocol": "1.66.0", + "@typescript-eslint/eslint-plugin": "8.4.0", + "@typescript-eslint/parser": "8.4.0", + "@vscode/debugadapter": "1.67.0", + "@vscode/debugprotocol": "1.67.0", "@vscode/extension-telemetry": "0.9.7", - "@vscode/vsce": "3.0.0", - "adm-zip": "0.5.15", + "@vscode/vsce": "3.1.0", + "adm-zip": "0.5.16", "antlr4": "4.13.2", "assert": "2.1.0", "copy-to-clipboard": "3.3.3", @@ -96,17 +105,20 @@ "esbuild-plugin-umd-wrapper": "3.0.0", "eslint": "8.57.0", "eslint-plugin-react-hooks": "4.6.2", + "langchain": "0.2.18", "mocha": "10.7.3", "npm-run-all": "4.1.5", "react": "17.0.2", "react-dom": "17.0.2", "rimraf": "6.0.1", + "run-script-os": "^1.1.6", "standard-version": "9.5.0", - "tslib": "2.6.3", + "tslib": "2.7.0", "typescript": "5.5.4", - "typescript-eslint": "8.2.0", + "typescript-eslint": "8.4.0", "use-resize-observer": "9.1.0", - "uuid": "10.0.0" + "uuid": "10.0.0", + "yargs": "^17.7.2" }, "author": { "name": "HPCC Systems®" @@ -1674,4 +1686,4 @@ } ] } -} \ No newline at end of file +} diff --git a/scripts/grammar-generate.sh b/scripts/grammar-generate.sh index 324cd04..48e5868 100644 --- a/scripts/grammar-generate.sh +++ b/scripts/grammar-generate.sh @@ -1,5 +1,9 @@ #!/bin/bash +# Latest Antlr 4.x: https://www.antlr.org/download.html +# Latest Kel Grammar: https://github.com/LexisNexis-RBA/Tardis/blob/master/kel-top/tools/kel-intellij-plugin/src/main/antlr/org/hpccsystems/intellij/language/plugin/parser/KELLexer.g4 +# Latest Salt Grammar: https://github.com/LexisNexis-RBA/Tardis/blob/master/SALT/src/main/java/com/relx/rba/tardis/salt/antlr/Salt.g4 + cd ./grammar/kel java -jar ../antlr-4.13.2-complete.jar -Dlanguage=JavaScript -o ../../src/grammar/kel -visitor -Xexact-output-dir ./*.g4 cd ../.. diff --git a/src/ecl/chat/index.ts b/src/ecl/chat/index.ts index 29716d5..05f295c 100644 --- a/src/ecl/chat/index.ts +++ b/src/ecl/chat/index.ts @@ -3,6 +3,7 @@ import { commands, getRandomGreeting } from "./constants"; import localize from "../../util/localize"; import { handleDocsCommand } from "./prompts/docs"; import { handleIssueManagement } from "./prompts/issues"; +import { checkModelExists } from "./utils/model"; const ECL_PARTICIPANT_ID = "chat.ecl"; @@ -28,8 +29,12 @@ function handleError(logger: vscode.TelemetryLogger, err: any, stream: vscode.Ch let eclChat: ECLChat; export class ECLChat { + modelPath: Promise; + protected constructor(ctx: vscode.ExtensionContext) { - const handler: vscode.ChatRequestHandler = async (request: vscode.ChatRequest, ctx: vscode.ChatContext, stream: vscode.ChatResponseStream, token: vscode.CancellationToken): Promise => { + this.modelPath = checkModelExists(ctx); + + const handler: vscode.ChatRequestHandler = async (request: vscode.ChatRequest, chatCtx: vscode.ChatContext, stream: vscode.ChatResponseStream, token: vscode.CancellationToken): Promise => { let cmdResult: any; stream.progress(localize(getRandomGreeting())); try { @@ -37,7 +42,7 @@ export class ECLChat { cmdResult = await handleIssueManagement(request, stream, token); logger.logUsage("request", { kind: commands.ISSUES }); } else { - cmdResult = await handleDocsCommand(request, stream, token); + cmdResult = await handleDocsCommand(request, stream, token, this.modelPath, vscode.Uri.joinPath(ctx.extensionUri, "dist", "docs.vecdb")); } } catch (err) { handleError(logger, err, stream); diff --git a/src/ecl/chat/prompts/docs.tsx b/src/ecl/chat/prompts/docs.tsx index 3647292..fd63df7 100644 --- a/src/ecl/chat/prompts/docs.tsx +++ b/src/ecl/chat/prompts/docs.tsx @@ -32,10 +32,10 @@ export class DocsPrompt extends PromptElement { } } -export async function handleDocsCommand(request: vscode.ChatRequest, stream: vscode.ChatResponseStream, token: vscode.CancellationToken): Promise<{ metadata: { command: string, hits: Hit[] } }> { +export async function handleDocsCommand(request: vscode.ChatRequest, stream: vscode.ChatResponseStream, token: vscode.CancellationToken, modelPath: Promise, docsPath: vscode.Uri): Promise<{ metadata: { command: string, hits: Hit[] } }> { const [model] = await vscode.lm.selectChatModels(MODEL_SELECTOR); if (model) { - const hits = await fetchContext(request.prompt); + const hits = await fetchContext(request.prompt, modelPath, docsPath); let promptProps: DocsPromptProps; if (!hits.length) { promptProps = { diff --git a/src/ecl/chat/utils/chatResponse.ts b/src/ecl/chat/utils/chatResponse.ts index 0cd86ed..8dff9fc 100644 --- a/src/ecl/chat/utils/chatResponse.ts +++ b/src/ecl/chat/utils/chatResponse.ts @@ -7,10 +7,10 @@ export interface PromptProps extends BasePromptElementProps { } export async function getChatResponse, P extends PromptProps>(prompt: T, promptProps: P, token: vscode.CancellationToken): Promise> { - const [model] = await vscode.lm.selectChatModels(MODEL_SELECTOR); - if (model) { - const { messages } = await renderPrompt(prompt, promptProps, { modelMaxPromptTokens: model.maxInputTokens }, model); - return await model.sendRequest(messages, {}, token); + const models = await vscode.lm.selectChatModels({ family: MODEL_SELECTOR.family, vendor: MODEL_SELECTOR.vendor }); + if (models.length) { + const { messages } = await renderPrompt(prompt, promptProps, { modelMaxPromptTokens: models[0].maxInputTokens }, models[0] as any); + return await models[0].sendRequest(messages, {}, token); } else { throw new Error("No model found"); } diff --git a/src/ecl/chat/utils/model.ts b/src/ecl/chat/utils/model.ts new file mode 100644 index 0000000..21d3456 --- /dev/null +++ b/src/ecl/chat/utils/model.ts @@ -0,0 +1,36 @@ +import * as vscode from "vscode"; +import * as path from "path"; +import { WebBlob } from "@hpcc-js/wasm-llama"; + +const MODEL_FILENAME = "bge-base-en-v1.5-q4_k_m.gguf"; +const MODEL_URL = `https://huggingface.co/CompendiumLabs/bge-base-en-v1.5-gguf/resolve/main/${MODEL_FILENAME}`; + +async function checkModelFolderExists(ctx: vscode.ExtensionContext) { + const modelsFolder = vscode.Uri.joinPath(ctx.globalStorageUri, "models"); + try { + await vscode.workspace.fs.stat(modelsFolder); + } catch (error: any) { + if (error.code === "FileNotFound") { + await vscode.workspace.fs.createDirectory(modelsFolder); + } else { + vscode.window.showErrorMessage(`Error locating models folder: ${error.message}`); + } + } + return modelsFolder; +} + +export async function checkModelExists(ctx: vscode.ExtensionContext) { + const modelsFolder = await checkModelFolderExists(ctx); + const modelPath = path.join(modelsFolder.fsPath, MODEL_FILENAME); + try { + await vscode.workspace.fs.stat(vscode.Uri.file(modelPath)); + } catch (error: any) { + if (error.code === "FileNotFound") { + const modelBlob = await WebBlob.create(new URL(MODEL_URL)); + await vscode.workspace.fs.writeFile(vscode.Uri.file(modelPath), new Uint8Array(await modelBlob.arrayBuffer())); + } else { + vscode.window.showErrorMessage(`Error locating model: ${error.message}`); + } + } + return vscode.Uri.file(modelPath); +} diff --git a/src/ecl/docs/SimilarityModel.ts b/src/ecl/docs/SimilarityModel.ts new file mode 100644 index 0000000..53aab3f --- /dev/null +++ b/src/ecl/docs/SimilarityModel.ts @@ -0,0 +1,51 @@ +import * as vscode from "vscode"; +import { Llama } from "@hpcc-js/wasm-llama"; +import { Zstd } from "@hpcc-js/wasm-zstd"; +import { Embeddings } from "@langchain/core/embeddings"; +import { MemoryVectorStore } from "langchain/vectorstores/memory"; + +class LlamaEmbeddings extends Embeddings { + + private _modelData: Promise; + + constructor(modelPath: Promise, fields?) { + super(fields); + this._modelData = modelPath + .then(modelPath => vscode.workspace.fs.readFile(modelPath)) + .then(modelBin => new Uint8Array(modelBin)) + ; + } + + embedDocuments(documents: string[]): Promise { + return Promise.all([Llama.load(), this._modelData]).then(([llama, modelData]) => { + const embeddings: number[][] = []; + for (const text of documents) { + const vectors = llama.embedding(text, modelData); + vectors.forEach(vector => { + embeddings.push(vector); + }); + Llama.unload(); + } + return embeddings; + }); + } + + embedQuery(document: string): Promise { + return Promise.all([Llama.load(), this._modelData]).then(([llama, modelData]) => { + const vectors = llama.embedding(document, modelData); + return vectors[0]; + }); + } +} + +export async function loadStore(modelPath: Promise, docsPath: vscode.Uri) { + const docsVStore = new MemoryVectorStore(new LlamaEmbeddings(modelPath)); + const [zstd, buff] = await Promise.all([Zstd.load(), vscode.workspace.fs.readFile(docsPath)]); + const jsonUint8Array = zstd.decompress(new Uint8Array(buff)); + const decoder = new TextDecoder(); + const jsonStr = decoder.decode(jsonUint8Array); + const data = JSON.parse(jsonStr); + docsVStore.memoryVectors = docsVStore.memoryVectors.concat(data.vectors); + return docsVStore; +} + diff --git a/src/ecl/docs/onlineHelp.ts b/src/ecl/docs/onlineHelp.ts index 6f405f1..2405458 100644 --- a/src/ecl/docs/onlineHelp.ts +++ b/src/ecl/docs/onlineHelp.ts @@ -1,9 +1,11 @@ import * as vscode from "vscode"; +import { loadStore } from "./SimilarityModel"; import { convert } from "html-to-text"; -export const ECLR_EN_US = "https://hpccsystems.com/wp-content/uploads/_documents/ECLR_EN_US"; -export const SLR_EN_US = "https://hpccsystems.com/wp-content/uploads/_documents/SLR_EN_US"; -export const ProgrammersGuide_EN_US = "https://hpccsystems.com/wp-content/uploads/_documents/ProgrammersGuide_EN_US"; +export const BASE_URL = "https://hpccsystems.com/wp-content/uploads/_documents"; +export const ECLR_EN_US = `${BASE_URL}/ECLR_EN_US`; +export const SLR_EN_US = `${BASE_URL}/SLR_EN_US`; +export const ProgrammersGuide_EN_US = `${BASE_URL}/ProgrammersGuide_EN_US`; interface HelpMeta { orig: string; @@ -62,46 +64,21 @@ export async function matchTopics(_word: string): Promise { }); } -const commonFillWords = [ - "the", "and", "for", "with", "from", "that", "this", "which", "what", "how", - "why", "when", "where", "who", "whom", "whose", "will", "would", "should", "could", - "can", "may", "might", "shall", "must", "have", "has", "had", "do", "does", "did", - "is", "are", "was", "were", "be", "been", "being", "it", "its", "they", "them", - "their", "our", "we", "us", "you", "your", "my", "mine", "his", "her", - "he", "she", "it", "its", "him", "her", "his", "hers", "they", "them", "their", - "theirs", "we", "us", "our", "ours", "you", "your", "yours", "my", "mine", - "our", "ours", "your", "yours", "my", "mine", "his", "her", "he", "she", "about", "know" -]; - -function simpleNormalize(text: string): string[] { - const s = text.toLocaleLowerCase().replace(/[^a-z0-9]/g, " "); - - const words = s.split(" ") - .filter((word) => word.length > 2) - .filter((word) => !commonFillWords.includes(word)) - .map((e) => e.trim()) - .filter((e) => !!e); - - return words; -} - export interface Hit { label: string; url: string; - exact: number; content: string; error?: string; } const _labelContentCache: { [id: string]: Promise } = {}; -function fetchContent(url: string, label: string, exact: number): Promise { +function fetchContent(url: string, label: string): Promise { if (!_labelContentCache[url]) { _labelContentCache[url] = fetch(url) .then(async response => { return { label, url, - exact, content: convert(await response.text(), { wordwrap: false }) }; }).catch(e => { @@ -109,7 +86,6 @@ function fetchContent(url: string, label: string, exact: number): Promise { label, url, content: "", - exact: 0, error: e.message() }; }); @@ -119,30 +95,22 @@ function fetchContent(url: string, label: string, exact: number): Promise { export async function fetchIndexes(): Promise { return Promise.all([ECLR_EN_US, SLR_EN_US, ProgrammersGuide_EN_US].map(url => { - return fetchContent(url, url, 0); + return fetchContent(url, url); })); } -export async function fetchContext(query: string): Promise { - - const terms = simpleNormalize(query); - const retVal: Promise[] = []; - await Promise.all(terms.map(term => { - let exact = 0; - return matchTopics(term) - .then(async matches => { - matches.forEach((match) => { - if (match.exact) { - exact++; - } - - const url = match.uri.toString(); - retVal.push(fetchContent(url, match.label, match.exact ? 1 : 0)); - }); - }); - })); - - return Promise.all(retVal).then(hits => { - return hits.sort((a, b) => b.exact - a.exact); +export async function fetchContext(query: string, modelPath: Promise, docsPath: vscode.Uri): Promise { + const docsVStore = await loadStore(modelPath, docsPath); + const similarPages = await docsVStore.similaritySearch(query, 5); + + return similarPages.map(similarPage => { + const parts = similarPage.metadata.url.split("/"); + const label = parts[parts.length - 1].split(".")[0]; + const url = `${BASE_URL}/${similarPage.metadata.url}`; + return { + label, + url, + content: similarPage.pageContent + }; }); } diff --git a/util/docs.vecdb b/util/docs.vecdb new file mode 100644 index 0000000..b76d7b5 Binary files /dev/null and b/util/docs.vecdb differ diff --git a/util/gather-docs.ts b/util/gather-docs.ts new file mode 100644 index 0000000..4bfc6ec --- /dev/null +++ b/util/gather-docs.ts @@ -0,0 +1,96 @@ +import { readdir, stat } from "node:fs/promises"; +import * as path from "node:path"; +import { spawn } from "node:child_process"; + +async function convertDocBookToMarkdown(inputFile: string): Promise { + return new Promise((resolve, reject) => { + const pandoc = spawn("pandoc", ["-f", "docbook", "-t", "markdown", "-s", inputFile]); + + let output: string = ""; + let errorOutput = ""; + + pandoc.stdout.on("data", (data) => { + output += data.toString(); + }); + + pandoc.stderr.on("data", (data) => { + errorOutput += data.toString(); + }); + + pandoc.on("close", (code) => { + if (code === 0) { + resolve(output); + } else { + reject(new Error(`Pandoc process exited with code ${code}: ${errorOutput}`)); + } + }); + }); +} + +async function urlExists(partialUrl: string): Promise { + try { + const response = await fetch(`https://hpccsystems.com/wp-content/uploads/_documents/${partialUrl}`); + if (response?.ok) { + return true; + } + } catch (e) { + console.error("Error checking URL:", e); + } + return false; +} + +async function* chunkMarkdown(md: string, urlPrefix: string) { + let url = ""; + let bookmark = ""; + let content = ""; + for (let line of md.split("\n")) { + const urlStart = line.indexOf("{#"); + let nextUrl = ""; + if (urlStart >= 0) { + const urlEnd = line.indexOf("}", urlStart); + nextUrl = line.substring(urlStart + 2, urlEnd); + line = line.substring(0, urlStart); + } + if (nextUrl) { + if (await urlExists(`${urlPrefix}/${nextUrl}.html`)) { + if (url && content) { + yield { url, bookmark, content }; + } + url = `${urlPrefix}/${nextUrl}.html`; + bookmark = ""; + content = ""; + } else if (url) { + if (url && content) { + yield { url, bookmark, content }; + } + bookmark = `#${nextUrl}`; + content = ""; + } else { + } + } + content += line + "\n"; + } + if (url && content) { + yield { url, bookmark, content }; + } +} + +export async function* findXmlFiles(dir: string, urlPrefix: string) { + + try { + const files = await readdir(dir); + + for (const file of files) { + const filePath = path.join(dir, file); + const fileStat = await stat(filePath); + + if (fileStat.isDirectory()) { + yield* findXmlFiles(filePath, urlPrefix); + } else if (filePath.endsWith(".xml")) { + yield* chunkMarkdown(await convertDocBookToMarkdown(filePath), urlPrefix); + } + } + } catch (e) { + console.error("Error reading directory:", e); + } +} diff --git a/util/index-docs.ts b/util/index-docs.ts new file mode 100644 index 0000000..e7c7b78 --- /dev/null +++ b/util/index-docs.ts @@ -0,0 +1,114 @@ +import path from "node:path"; +import { writeFile, stat } from "node:fs/promises"; +import { homedir } from "node:os"; +import * as yargs from "yargs"; +import { hideBin } from "yargs/helpers"; +import { MemoryVectorStore } from "langchain/vectorstores/memory"; +import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters"; +import { OllamaEmbeddings } from "@langchain/ollama"; +import { Zstd } from "@hpcc-js/wasm-zstd"; +import { findXmlFiles } from "./gather-docs"; + +const myYargs = yargs.default(hideBin(process.argv)) as yargs.Argv<{}>; +myYargs + .usage("Usage: node ./dist-util/index-docs.mjs [options] path/to/hpcc-platfornm/docs") + .demandCommand(0, 1) + .example("node ./dist-util/index-docs.mjs --dry-run ~/HPCC-Platform/docs") + .alias("d", "dry-run") + .boolean("d") + .describe("d", "Dry run (won't write any output files).") + .help("h") + .alias("h", "help") + .epilog("https://github.com/hpcc-systems/vscode-ecl") + ; + +const argv = await myYargs.argv; + +class OllamaEmbeddingsEx extends OllamaEmbeddings { + + constructor(fields?) { + super(fields); + } + + embedDocuments(documents: string[]): Promise { + return super.embedDocuments(documents).then(embeddings => { + return embeddings; + }); + } + + embedQuery(document: string): Promise { + return super.embedQuery(document).then(embedding => { + return embedding; + }); + } +} + +function serializeStore(store: MemoryVectorStore): string { + const serializeData = { + vectors: store.memoryVectors, + vserion: 1, + isMemoryVectorStore: true + }; + return JSON.stringify(serializeData); +} + +async function saveStore(store: MemoryVectorStore, path: string) { + const zstd = await Zstd.load(); + const str = serializeStore(store); + const encoder = new TextEncoder(); + const encodedString = encoder.encode(str); + const compressed = zstd.compress(encodedString); + return writeFile(path, compressed); +} + +async function calcEmbeddings(vectorStore: MemoryVectorStore, dir: string, urlPrefix: string) { + const splitter = new RecursiveCharacterTextSplitter(); + for await (const file of findXmlFiles(path.resolve(dir), urlPrefix)) { + console.log(`https://hpccsystems.com/wp-content/uploads/_documents/${file.url}${file.bookmark}`); + const childDocs = await splitter.splitText(file.content); + await Promise.all(childDocs.map(childDoc => { + console.log("Calculating embedding...", childDoc.length); + return vectorStore.addDocuments([{ + metadata: { + url: file.url, + }, + pageContent: childDoc + }]); + })); + } +} + +try { + let docsFolder = argv._[0] as string; + if (docsFolder[0] === "~") { + docsFolder = path.join(homedir(), docsFolder.slice(1)); + } + docsFolder = path.resolve(docsFolder); + const fileStat = await stat(docsFolder); + if (!fileStat.isDirectory()) { + throw new Error("Docs path does not exist."); + } + + const ollamaEmbedding = new OllamaEmbeddingsEx({ + model: "bge-base-en-v1.5-q4_k_m", + truncate: true + }); + + const vectorStore = new MemoryVectorStore(ollamaEmbedding); + await Promise.all([ + calcEmbeddings(vectorStore, `${docsFolder}/EN_US/ECLProgrammersGuide/PRG_Mods`, "ProgrammersGuide_EN_US"), + calcEmbeddings(vectorStore, `${docsFolder}/EN_US/ECLLanguageReference/ECLR_mods`, "ECLR_EN_US"), + calcEmbeddings(vectorStore, `${docsFolder}/EN_US/ECLStandardLibraryReference/SLR-Mods`, "SLR_EN_US"), + ]); + + if (!argv["dry-run"]) { + saveStore(vectorStore, "./util/docs.vecdb"); + } +} catch (e) { + console.error("Error:", e.message, "\n"); + myYargs.showHelp(); + process.exit(1); +} finally { + process.exit(0); +} +