refactor: use native uuid generator.

apps/api/package.json

@@ -113,7 +113,6 @@
     "turndown-plugin-gfm": "^1.0.2",
     "typesense": "^1.5.4",
     "unstructured-client": "^0.11.3",
-    "uuid": "^10.0.0",
     "wordpos": "^2.1.0",
     "ws": "^8.18.0",
     "xml2js": "^0.6.2",

apps/api/src/__tests__/e2e_full_withAuth/index.test.ts

@@ -1,6 +1,5 @@
 import request from "supertest";
 import dotenv from "dotenv";
-import { v4 as uuidv4 } from "uuid";
 
 dotenv.config();
 
@@ -396,7 +395,7 @@ describe("E2E Tests for API Routes", () => {
       );
     });
     it.concurrent('should prevent duplicate requests using the same idempotency key', async () => {
-      const uniqueIdempotencyKey = uuidv4();
+      const uniqueIdempotencyKey = crypto.randomUUID();
 
       // First request with the idempotency key
       const firstResponse = await request(TEST_URL)

apps/api/src/controllers/__tests__/crawl.test.ts

@@ -3,7 +3,6 @@ import { Request, Response } from 'express';
 import { authenticateUser } from '../auth'; // Ensure this import is correct
 import { createIdempotencyKey } from '../../services/idempotency/create';
 import { validateIdempotencyKey } from '../../services/idempotency/validate';
-import { v4 as uuidv4 } from 'uuid';
 
 jest.mock('../auth', () => ({
   authenticateUser: jest.fn().mockResolvedValue({
@@ -20,7 +19,7 @@ describe('crawlController', () => {
   it('should prevent duplicate requests using the same idempotency key', async () => {
     const req = {
       headers: {
-        'x-idempotency-key': await uuidv4(),
+        'x-idempotency-key': crypto.randomUUID(),
         'Authorization': `Bearer ${process.env.TEST_API_KEY}`
       },
       body: {
@@ -44,4 +43,4 @@ describe('crawlController', () => {
     expect(res.status).toHaveBeenCalledWith(409);
     expect(res.json).toHaveBeenCalledWith({ error: 'Idempotency key already used' });
   });
-});
+});

apps/api/src/controllers/auth.ts

@@ -32,9 +32,11 @@ import { AuthCreditUsageChunk } from "./v1/types";
 //     .eq('key', normalizedApi)
 //     .limit(1)
 //     .single();
+const UUID_REGEX = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$/i;
+
 function normalizedApiIsUuid(potentialUuid: string): boolean {
   // Check if the string is a valid UUID
-  return validate(potentialUuid);
+  return typeof potentialUuid === 'string' && UUID_REGEX.test(potentialUuid);
 }
 
 export async function setCachedACUC(

apps/api/src/controllers/v0/crawl.ts

@@ -8,7 +8,6 @@ import { logCrawl } from "../../../src/services/logging/crawl_log";
 import { validateIdempotencyKey } from "../../../src/services/idempotency/validate";
 import { createIdempotencyKey } from "../../../src/services/idempotency/create";
 import { defaultCrawlPageOptions, defaultCrawlerOptions, defaultOrigin } from "../../../src/lib/default-values";
-import { v4 as uuidv4 } from "uuid";
 import { Logger } from "../../../src/lib/logger";
 import { addCrawlJob, addCrawlJobs, crawlToCrawler, lockURL, lockURLs, saveCrawl, StoredCrawl } from "../../../src/lib/crawl-redis";
 import { getScrapeQueue } from "../../../src/services/queue-service";
@@ -103,7 +102,7 @@ export async function crawlController(req: Request, res: Response) {
     //   try {
     //     const a = new WebScraperDataProvider();
     //     await a.setOptions({
-    //       jobId: uuidv4(),
+    //       jobId: crypto.randomUUID(),
     //       mode: "single_urls",
     //       urls: [url],
     //       crawlerOptions: { ...crawlerOptions, returnOnlyUrls: true },
@@ -128,7 +127,7 @@ export async function crawlController(req: Request, res: Response) {
     //   }
     // }
 
-    const id = uuidv4();
+    const id = crypto.randomUUID();
 
     await logCrawl(id, team_id);
 
@@ -164,7 +163,7 @@ export async function crawlController(req: Request, res: Response) {
       }
       const jobs = sitemap.map((x) => {
         const url = x.url;
-        const uuid = uuidv4();
+        const uuid = crypto.randomUUID();
         return {
           name: uuid,
           data: {

apps/api/src/controllers/v0/crawlPreview.ts

@@ -2,7 +2,6 @@ import { Request, Response } from "express";
 import { authenticateUser } from "../auth";
 import { RateLimiterMode } from "../../../src/types";
 import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist";
-import { v4 as uuidv4 } from "uuid";
 import { Logger } from "../../../src/lib/logger";
 import { addCrawlJob, crawlToCrawler, lockURL, saveCrawl, StoredCrawl } from "../../../src/lib/crawl-redis";
 import { addScrapeJob } from "../../../src/services/queue-jobs";
@@ -51,7 +50,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
     //   try {
     //     const a = new WebScraperDataProvider();
     //     await a.setOptions({
-    //       jobId: uuidv4(),
+    //       jobId: crypto.randomUUID(),
     //       mode: "single_urls",
     //       urls: [url],
     //       crawlerOptions: { ...crawlerOptions, returnOnlyUrls: true },
@@ -76,7 +75,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
     //   }
     // }
 
-    const id = uuidv4();
+    const id = crypto.randomUUID();
 
     let robots;
 

apps/api/src/controllers/v0/scrape.ts

@@ -18,7 +18,6 @@ import {
 } from "../../lib/default-values";
 import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
 import { getScrapeQueue } from "../../services/queue-service";
-import { v4 as uuidv4 } from "uuid";
 import { Logger } from "../../lib/logger";
 import * as Sentry from "@sentry/node";
 import { getJobPriority } from "../../lib/job-priority";
@@ -208,7 +207,7 @@ export async function scrapeController(req: Request, res: Response) {
       });
     }
 
-    const jobId = uuidv4();
+    const jobId = crypto.randomUUID();
 
     const startTime = new Date().getTime();
     const result = await scrapeHelper(

apps/api/src/controllers/v0/search.ts

@@ -7,7 +7,6 @@ import { logJob } from "../../services/logging/log_job";
 import { PageOptions, SearchOptions } from "../../lib/entities";
 import { search } from "../../search";
 import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import { v4 as uuidv4 } from "uuid";
 import { Logger } from "../../lib/logger";
 import { getScrapeQueue } from "../../services/queue-service";
 import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
@@ -82,7 +81,7 @@ export async function searchHelper(
 
   const jobDatas = res.map(x => {
     const url = x.url;
-    const uuid = uuidv4();
+    const uuid = crypto.randomUUID();
     return {
       name: uuid,
       data: {
@@ -157,7 +156,7 @@ export async function searchController(req: Request, res: Response) {
 
     const searchOptions = req.body.searchOptions ?? { limit: 5 };
 
-    const jobId = uuidv4();
+    const jobId = crypto.randomUUID();
 
     try {
       const { success: creditsCheckSuccess, message: creditsCheckMessage } =

apps/api/src/controllers/v1/__tests__/crawl.test.ts.WIP

@@ -3,7 +3,6 @@ import { Request, Response } from 'express';
 import { authenticateUser } from '../auth'; // Ensure this import is correct
 import { createIdempotencyKey } from '../../services/idempotency/create';
 import { validateIdempotencyKey } from '../../services/idempotency/validate';
-import { v4 as uuidv4 } from 'uuid';
 
 jest.mock('../auth', () => ({
   authenticateUser: jest.fn().mockResolvedValue({
@@ -20,7 +19,7 @@ describe('crawlController', () => {
   it('should prevent duplicate requests using the same idempotency key', async () => {
     const req = {
       headers: {
-        'x-idempotency-key': await uuidv4(),
+        'x-idempotency-key': crypto.randomUUID(),
         'Authorization': `Bearer ${process.env.TEST_API_KEY}`
       },
       body: {
@@ -44,4 +43,4 @@ describe('crawlController', () => {
     expect(res.status).toHaveBeenCalledWith(409);
     expect(res.json).toHaveBeenCalledWith({ error: 'Idempotency key already used' });
   });
-});
+});

apps/api/src/controllers/v1/batch-scrape.ts

@@ -1,5 +1,4 @@
 import { Response } from "express";
-import { v4 as uuidv4 } from "uuid";
 import {
   BatchScrapeRequest,
   batchScrapeRequestSchema,
@@ -24,7 +23,7 @@ export async function batchScrapeController(
 ) {
   req.body = batchScrapeRequestSchema.parse(req.body);
 
-  const id = uuidv4();
+  const id = crypto.randomUUID();
 
   await logCrawl(id, req.auth.team_id);
 
@@ -58,7 +57,7 @@ export async function batchScrapeController(
   }
 
   const jobs = req.body.urls.map((x) => {
-    const uuid = uuidv4();
+    const uuid = crypto.randomUUID();
     return {
       name: uuid,
       data: {

apps/api/src/controllers/v1/crawl-status-ws.ts

@@ -3,7 +3,6 @@ import { RateLimiterMode } from "../../types";
 import { authenticateUser } from "../auth";
 import { CrawlStatusParams, CrawlStatusResponse, Document, ErrorResponse, legacyDocumentConverter, RequestWithAuth } from "./types";
 import { WebSocket } from "ws";
-import { v4 as uuidv4 } from "uuid";
 import { Logger } from "../../lib/logger";
 import { getCrawl, getCrawlExpiry, getCrawlJobs, getDoneJobsOrdered, getDoneJobsOrderedLength, getThrottledJobs, isCrawlFinished, isCrawlFinishedLocked } from "../../lib/crawl-redis";
 import { getScrapeQueue } from "../../services/queue-service";
@@ -158,7 +157,7 @@ export async function crawlStatusWSController(ws: WebSocket, req: RequestWithAut
   } catch (err) {
     Sentry.captureException(err);
 
-    const id = uuidv4();
+    const id = crypto.randomUUID();
     let verbose = JSON.stringify(err);
     if (verbose === "{}") {
       if (err instanceof Error) {

apps/api/src/controllers/v1/crawl.ts

@@ -1,5 +1,4 @@
 import { Response } from "express";
-import { v4 as uuidv4 } from "uuid";
 import {
   CrawlRequest,
   crawlRequestSchema,
@@ -30,7 +29,7 @@ export async function crawlController(
 ) {
   req.body = crawlRequestSchema.parse(req.body);
 
-  const id = uuidv4();
+  const id = crypto.randomUUID();
 
   await logCrawl(id, req.auth.team_id);
 
@@ -103,7 +102,7 @@ export async function crawlController(
       }
     const jobs = sitemap.map((x) => {
       const url = x.url;
-      const uuid = uuidv4();
+      const uuid = crypto.randomUUID();
       return {
         name: uuid,
         data: {

apps/api/src/controllers/v1/map.ts

@@ -1,5 +1,4 @@
 import { Response } from "express";
-import { v4 as uuidv4 } from "uuid";
 import {
   legacyCrawlerOptions,
   mapRequestSchema,
@@ -39,7 +38,7 @@ export async function mapController(
 
   const limit: number = req.body.limit ?? MAX_MAP_LIMIT;
 
-  const id = uuidv4();
+  const id = crypto.randomUUID();
   let links: string[] = [req.body.url];
 
   const sc: StoredCrawl = {

apps/api/src/controllers/v1/scrape.ts

@@ -11,7 +11,6 @@ import {
   ScrapeResponse,
 } from "./types";
 import { billTeam } from "../../services/billing/credit_billing";
-import { v4 as uuidv4 } from "uuid";
 import { numTokensFromString } from "../../lib/LLM-extraction/helpers";
 import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
 import { logJob } from "../../services/logging/log_job";
@@ -29,7 +28,7 @@ export async function scrapeController(
   const timeout = req.body.timeout;
   const pageOptions = legacyScrapeOptions(req.body);
   const extractorOptions = req.body.extract ? legacyExtractorOptions(req.body.extract) : undefined;
-  const jobId = uuidv4();
+  const jobId = crypto.randomUUID();
 
   const startTime = new Date().getTime();
   const jobPriority = await getJobPriority({

apps/api/src/index.ts

@@ -19,7 +19,6 @@ import expressWs from "express-ws";
 import { crawlStatusWSController } from "./controllers/v1/crawl-status-ws";
 import { ErrorResponse, ResponseWithSentry } from "./controllers/v1/types";
 import { ZodError } from "zod";
-import { v4 as uuidv4 } from "uuid";
 import dns from 'node:dns';
 
 const { createBullBoard } = require("@bull-board/api");
@@ -194,7 +193,7 @@ app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response
     return res.status(400).json({ success: false, error: 'Bad request, malformed JSON' });
   }
 
-  const id = res.sentry ?? uuidv4();
+  const id = res.sentry ?? crypto.randomUUID();
   let verbose = JSON.stringify(err);
   if (verbose === "{}") {
     if (err instanceof Error) {

apps/api/src/run-req.ts

@@ -1,6 +1,5 @@
 import axios from "axios";
 import { promises as fs } from "fs";
-import { v4 as uuidV4 } from "uuid";
 
 interface Result {
   start_url: string;
@@ -10,7 +9,7 @@ interface Result {
 }
 
 async function sendCrawl(result: Result): Promise<string | undefined> {
-  const idempotencyKey = uuidV4();
+  const idempotencyKey = crypto.randomUUID();
   const url = result.start_url;
   try {
     const response = await axios.post(

apps/api/src/services/queue-jobs.ts

@@ -1,6 +1,5 @@
 import { Job, Queue } from "bullmq";
 import { getScrapeQueue } from "./queue-service";
-import { v4 as uuidv4 } from "uuid";
 import { WebScraperOptions } from "../types";
 import * as Sentry from "@sentry/node";
 
@@ -20,7 +19,7 @@ async function addScrapeJobRaw(
 export async function addScrapeJob(
   webScraperOptions: WebScraperOptions,
   options: any = {},
-  jobId: string = uuidv4(),
+  jobId: string = crypto.randomUUID(),
   jobPriority: number = 10
 ): Promise<Job> {
 

apps/api/src/services/queue-worker.ts

@@ -16,7 +16,6 @@ import { Job, Queue } from "bullmq";
 import { Logger } from "../lib/logger";
 import { Worker } from "bullmq";
 import systemMonitor from "./system-monitor";
-import { v4 as uuidv4 } from "uuid";
 import {
   addCrawlJob,
   addCrawlJobDone,
@@ -125,7 +124,7 @@ const workerFun = async (
       console.log("No longer accepting new jobs. SIGINT");
       break;
     }
-    const token = uuidv4();
+    const token = crypto.randomUUID();
     const canAcceptConnection = await monitor.acceptConnection();
     if (!canAcceptConnection) {
       console.log("Cant accept connection");
@@ -384,7 +383,7 @@ async function processJob(job: Job, token: string) {
                 team_id: sc.team_id,
                 basePriority: job.data.crawl_id ? 20 : 10,
               });
-              const jobId = uuidv4();
+              const jobId = crypto.randomUUID();
 
               // console.log("plan: ",  sc.plan);
               // console.log("team_id: ", sc.team_id)

apps/js-sdk/firecrawl/package.json

@@ -36,7 +36,6 @@
   },
   "homepage": "https://github.com/mendableai/firecrawl#readme",
   "devDependencies": {
-    "uuid": "^9.0.1",
     "dotenv": "^16.4.5",
     "@jest/globals": "^29.7.0",
     "@types/axios": "^0.14.0",

apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts

@@ -6,7 +6,6 @@ import FirecrawlApp, {
   ScrapeResponseV0,
   SearchResponseV0,
 } from "../../index";
-import { v4 as uuidv4 } from "uuid";
 import dotenv from "dotenv";
 import { describe, test, expect } from "@jest/globals";
 
@@ -200,7 +199,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {
       apiUrl: API_URL,
       version: "v0",
     });
-    const uniqueIdempotencyKey = uuidv4();
+    const uniqueIdempotencyKey = crypto.randomUUID();
     const response = (await app.crawlUrl(
       "https://roastmywebsite.ai",
       { crawlerOptions: { excludes: ["blog/*"] } },