feat: add vision to openai (#304)

admin/actions/code/vision.ts

@@ -0,0 +1,12 @@
+import { type Props } from "../../../openai/loaders/vision.ts";
+import { AppContext } from "../../mod.ts";
+
+export default async function action(
+  props: Props,
+  _req: Request,
+  ctx: AppContext,
+): Promise<string | null> {
+  const response = await ctx.invoke("openai/loaders/vision.ts", props);
+
+  return response.choices?.[0]?.message.content;
+}

admin/manifest.gen.ts

@@ -19,15 +19,16 @@ import * as $$$$$$$$$2 from "./actions/blocks/safeDelete.ts";
 import * as $$$$$$$$$3 from "./actions/blocks/newRevision.ts";
 import * as $$$$$$$$$4 from "./actions/blocks/delete.ts";
 import * as $$$$$$$$$5 from "./actions/releases/fork.ts";
-import * as $$$$$$$$$6 from "./actions/code/suggestions.ts";
-import * as $$$$$$$$$7 from "./actions/sites/linkRepo.ts";
-import * as $$$$$$$$$8 from "./actions/sites/newDomain.ts";
-import * as $$$$$$$$$9 from "./actions/sites/unlinkRepo.ts";
-import * as $$$$$$$$$10 from "./actions/github/setStatus.ts";
-import * as $$$$$$$$$11 from "./actions/github/webhooks/broker.ts";
-import * as $$$$$$$$$12 from "./actions/pages/publish.ts";
-import * as $$$$$$$$$13 from "./actions/pages/new.ts";
-import * as $$$$$$$$$14 from "./actions/pages/delete.ts";
+import * as $$$$$$$$$6 from "./actions/code/vision.ts";
+import * as $$$$$$$$$7 from "./actions/code/suggestions.ts";
+import * as $$$$$$$$$8 from "./actions/sites/linkRepo.ts";
+import * as $$$$$$$$$9 from "./actions/sites/newDomain.ts";
+import * as $$$$$$$$$10 from "./actions/sites/unlinkRepo.ts";
+import * as $$$$$$$$$11 from "./actions/github/setStatus.ts";
+import * as $$$$$$$$$12 from "./actions/github/webhooks/broker.ts";
+import * as $$$$$$$$$13 from "./actions/pages/publish.ts";
+import * as $$$$$$$$$14 from "./actions/pages/new.ts";
+import * as $$$$$$$$$15 from "./actions/pages/delete.ts";
 
 const manifest = {
   "loaders": {
@@ -49,16 +50,17 @@ const manifest = {
     "deco-sites/admin/actions/blocks/publish.ts": $$$$$$$$$0,
     "deco-sites/admin/actions/blocks/restore.ts": $$$$$$$$$1,
     "deco-sites/admin/actions/blocks/safeDelete.ts": $$$$$$$$$2,
-    "deco-sites/admin/actions/code/suggestions.ts": $$$$$$$$$6,
-    "deco-sites/admin/actions/github/setStatus.ts": $$$$$$$$$10,
-    "deco-sites/admin/actions/github/webhooks/broker.ts": $$$$$$$$$11,
-    "deco-sites/admin/actions/pages/delete.ts": $$$$$$$$$14,
-    "deco-sites/admin/actions/pages/new.ts": $$$$$$$$$13,
-    "deco-sites/admin/actions/pages/publish.ts": $$$$$$$$$12,
+    "deco-sites/admin/actions/code/suggestions.ts": $$$$$$$$$7,
+    "deco-sites/admin/actions/code/vision.ts": $$$$$$$$$6,
+    "deco-sites/admin/actions/github/setStatus.ts": $$$$$$$$$11,
+    "deco-sites/admin/actions/github/webhooks/broker.ts": $$$$$$$$$12,
+    "deco-sites/admin/actions/pages/delete.ts": $$$$$$$$$15,
+    "deco-sites/admin/actions/pages/new.ts": $$$$$$$$$14,
+    "deco-sites/admin/actions/pages/publish.ts": $$$$$$$$$13,
     "deco-sites/admin/actions/releases/fork.ts": $$$$$$$$$5,
-    "deco-sites/admin/actions/sites/linkRepo.ts": $$$$$$$$$7,
-    "deco-sites/admin/actions/sites/newDomain.ts": $$$$$$$$$8,
-    "deco-sites/admin/actions/sites/unlinkRepo.ts": $$$$$$$$$9,
+    "deco-sites/admin/actions/sites/linkRepo.ts": $$$$$$$$$8,
+    "deco-sites/admin/actions/sites/newDomain.ts": $$$$$$$$$9,
+    "deco-sites/admin/actions/sites/unlinkRepo.ts": $$$$$$$$$10,
   },
   "name": "deco-sites/admin",
   "baseUrl": import.meta.url,

admin/mod.ts

@@ -15,6 +15,7 @@ import { pushEventHandler } from "./github/push.ts";
 import { State as Resolvables } from "./loaders/state.ts";
 import manifest, { Manifest as AppManifest } from "./manifest.gen.ts";
 import { Manifest as AIAssistantManifest } from "../ai-assistants/manifest.gen.ts";
+import { Manifest as OpenAIManifest } from "../openai/manifest.gen.ts";
 
 export const ANONYMOUS = "Anonymous";
 export interface BlockStore extends Release {
@@ -125,7 +126,9 @@ export default function App(
 }
 
 export type AppContext = AC<
-  Omit<App<AIAssistantManifest>, "dependencies"> & ReturnType<typeof App>
+  & Omit<App<AIAssistantManifest>, "dependencies">
+  & Omit<App<OpenAIManifest>, "dependencies">
+  & ReturnType<typeof App>
 >;
 
 export type Manifest = ManifestOf<ReturnType<typeof App>>;

openai/loaders/vision.ts

@@ -0,0 +1,46 @@
+import { AppContext } from "../mod.ts";
+import { OpenAI } from "../deps.ts";
+
+export interface Props {
+  /**
+   * @description the links of the site that contains images
+   * @examples "https://www.instagram.com/marcoscandeia"
+   */
+  images: string[];
+  /**
+   * @description what kind of description do you want?
+   * @examples you can ask for something like "What’s in this image?"\n or "Is there a car?"
+   */
+  prompt: string;
+  choices?: number;
+  maxTokens?: number;
+}
+
+export type Return = OpenAI.ChatCompletion;
+
+export default async function (
+  { images, prompt, choices = 1, maxTokens = 4096 }: Props,
+  _req: Request,
+  ctx: AppContext,
+): Promise<Return> {
+  const response = await ctx.openAI.chat.completions.create({
+    n: choices,
+    stream: false,
+    max_tokens: maxTokens,
+    model: "gpt-4-vision-preview",
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "text", text: prompt },
+          ...images.map((image) => ({
+            type: "image_url" as const,
+            image_url: { "url": image },
+          })),
+        ],
+      },
+    ],
+  });
+
+  return response;
+}

openai/manifest.gen.ts

@@ -2,7 +2,12 @@
 // This file SHOULD be checked into source version control.
 // This file is automatically updated during development when running `dev.ts`.
 
+import * as $$$0 from "./loaders/vision.ts";
+
 const manifest = {
+  "loaders": {
+    "openai/loaders/vision.ts": $$$0,
+  },
   "name": "openai",
   "baseUrl": import.meta.url,
 };