From 31aae1c11be4cf0541aaad1904fb2c98a37af95c Mon Sep 17 00:00:00 2001
From: Sebastien Flory <sflory@gmail.com>
Date: Fri, 29 Nov 2024 14:09:40 +0100
Subject: [PATCH] Increase size of images exposed to models

---
 front/lib/api/files/upload.ts | 15 +++++++++++----
 front/package-lock.json       | 29 -----------------------------
 2 files changed, 11 insertions(+), 33 deletions(-)

diff --git a/front/lib/api/files/upload.ts b/front/lib/api/files/upload.ts
index 370ec1622aaf..6a46c3cc2e1f 100644
--- a/front/lib/api/files/upload.ts
+++ b/front/lib/api/files/upload.ts
@@ -81,10 +81,17 @@ const resizeAndUploadToFileStorage: ProcessingFunction = async (
     version: "original",
   });
 
-  // Resize the image, preserving the aspect ratio. Longest side is max 768px.
-  const resizedImageStream = sharp().resize(768, 768, {
-    fit: sharp.fit.inside, // Ensure longest side is 768px.
-    withoutEnlargement: true, // Avoid upscaling if image is smaller than 768px.
+  // Anthropic https://docs.anthropic.com/en/docs/build-with-claude/vision#evaluate-image-size
+  // OpenAI https://platform.openai.com/docs/guides/vision#calculating-costs
+
+  // Anthropic recommends <= 1568px on any side.
+  // OpenAI recommends <= 2048px on the longuest side, 768px on the shortest side.
+
+  // Resize the image, preserving the aspect ratio based on the longest side compatible with both models.
+  // In case of GPT, it might incure a resize on their side as well but doing the math here would mean downloading the file first instead of streaming it.
+  const resizedImageStream = sharp().resize(1568, 1568, {
+    fit: sharp.fit.inside, // Ensure longest side is 1568px.
+    withoutEnlargement: true, // Avoid upscaling if image is smaller than 1568px.
   });
 
   const writeStream = file.getWriteStream({
diff --git a/front/package-lock.json b/front/package-lock.json
index f5af4b874c46..7d835097f088 100644
--- a/front/package-lock.json
+++ b/front/package-lock.json
@@ -76,7 +76,6 @@
         "next": "^14.2.3",
         "next-swagger-doc": "^0.4.0",
         "openai": "^4.28.0",
-        "pdfjs-dist": "^4.2.67",
         "pegjs": "^0.10.0",
         "pg": "^8.8.0",
         "pg-hstore": "^2.3.4",
@@ -141,7 +140,6 @@
         "@types/luxon": "^3.4.2",
         "@types/minimist": "^1.2.2",
         "@types/node": "^20.12.12",
-        "@types/pdfjs-dist": "^2.10.378",
         "@types/pegjs": "^0.10.3",
         "@types/react": "^18.3.11",
         "@types/react-dom": "^18.3.0",
@@ -18839,14 +18837,6 @@
       "license": "MIT",
       "peer": true
     },
-    "node_modules/@types/pdfjs-dist": {
-      "version": "2.10.378",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "pdfjs-dist": "*"
-      }
-    },
     "node_modules/@types/pegjs": {
       "version": "0.10.4",
       "dev": true,
@@ -29624,25 +29614,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/path2d": {
-      "version": "0.2.0",
-      "license": "MIT",
-      "optional": true,
-      "engines": {
-        "node": ">=6"
-      }
-    },
-    "node_modules/pdfjs-dist": {
-      "version": "4.2.67",
-      "license": "Apache-2.0",
-      "engines": {
-        "node": ">=18"
-      },
-      "optionalDependencies": {
-        "canvas": "^2.11.2",
-        "path2d": "^0.2.0"
-      }
-    },
     "node_modules/peek-readable": {
       "version": "5.3.1",
       "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.3.1.tgz",