feat: add support for multimodal in Vertex

README.md

@@ -762,21 +762,33 @@ MODELS=`[
     {
        "name": "gemini-1.5-pro",
        "displayName": "Vertex Gemini Pro 1.5",
+       "multimodal": true,
        "endpoints" : [{
           "type": "vertex",
           "project": "abc-xyz",
           "location": "europe-west3",
-          "model": "gemini-1.5-pro-preview-0409", // model-name
+          "model": {
+            "id": "some-id",
+            "name": "gemini-1.5-pro-preview-0409"
+          },
 
           // Optional
           "safetyThreshold": "BLOCK_MEDIUM_AND_ABOVE",
           "apiEndpoint": "", // alternative api endpoint url,
-          // Optional
           "tools": [{
             "googleSearchRetrieval": {
               "disableAttribution": true
             }
-          }]
+          }],
+          "multimodal": "{
+            "image": {
+              "supportedMimeTypes": ["image/png", "image/jpeg", "image/webp"],
+              "preferredMimeType": "image/png",
+              "maxSizeInMB": 5,
+              "maxWidth": 2000,
+              "maxHeight": number;
+            }
+          }"
        }]
      },
 ]`

src/lib/server/endpoints/google/endpointVertex.ts

@@ -9,6 +9,7 @@ import type { Endpoint } from "../endpoints";
 import { z } from "zod";
 import type { Message } from "$lib/types/Message";
 import type { TextGenerationStreamOutput } from "@huggingface/inference";
+import { createImageProcessorOptionsValidator, makeImageProcessor } from "../images";
 
 export const endpointVertexParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -27,10 +28,28 @@ export const endpointVertexParametersSchema = z.object({
 		])
 		.optional(),
 	tools: z.array(z.any()).optional(),
+	multimodal: z
+		.object({
+			image: createImageProcessorOptionsValidator({
+				supportedMimeTypes: [
+					"image/png",
+					"image/jpeg",
+					"image/webp",
+					"image/avif",
+					"image/tiff",
+					"image/gif",
+				],
+				preferredMimeType: "image/webp",
+				maxSizeInMB: Infinity,
+				maxWidth: 4096,
+				maxHeight: 4096,
+			}),
+		})
+		.default({}),
 });
 
 export function endpointVertex(input: z.input<typeof endpointVertexParametersSchema>): Endpoint {
-	const { project, location, model, apiEndpoint, safetyThreshold, tools } =
+	const { project, location, model, apiEndpoint, safetyThreshold, tools, multimodal } =
 		endpointVertexParametersSchema.parse(input);
 
 	const vertex_ai = new VertexAI({
@@ -43,7 +62,7 @@ export function endpointVertex(input: z.input<typeof endpointVertexParametersSch
 		const parameters = { ...model.parameters, ...generateSettings };
 
 		const generativeModel = vertex_ai.getGenerativeModel({
-			model: model.id ?? model.name,
+			model: model.name ?? model.id,
 			safetySettings: safetyThreshold
 				? [
 						{
@@ -83,16 +102,30 @@ export function endpointVertex(input: z.input<typeof endpointVertexParametersSch
 			messages.shift();
 		}
 
-		const vertexMessages = messages.map(({ from, content }: Omit<Message, "id">): Content => {
-			return {
-				role: from === "user" ? "user" : "model",
-				parts: [
-					{
-						text: content,
-					},
-				],
-			};
-		});
+		const vertexMessages = await Promise.all(
+			messages.map(async ({ from, content, files }: Omit<Message, "id">): Promise<Content> => {
+				const imageProcessor = makeImageProcessor(multimodal.image);
+				const processedFiles =
+					files && files.length > 0
+						? await Promise.all(files.map(async (file) => imageProcessor(file)))
+						: [];
+
+				return {
+					role: from === "user" ? "user" : "model",
+					parts: [
+						...processedFiles.map((processedFile) => ({
+							inlineData: {
+								data: processedFile.image.toString("base64"),
+								mimeType: processedFile.mime,
+							},
+						})),
+						{
+							text: content,
+						},
+					],
+				};
+			})
+		);
 
 		const result = await generativeModel.generateContentStream({
 			contents: vertexMessages,