Update defImages function to support multiple file types and handle Blob conversion

docs/src/content/docs/reference/scripts/images.md

@@ -15,3 +15,29 @@
 ```
 
 Read more about [OpenAI Vision](https://platform.openai.com/docs/guides/vision/limitations).
+
+## URLs
+
+Public URLs (that do not require authentication) will be passed directly to OpenAI.
+
+```js
+defImages(
+    "https://github.com/microsoft/genaiscript/blob/main/docs/public/images/logo.png?raw=true"
+)
+```
+
+Local files are loaded and encoded as a data uri.
+
+## Buffer, Blob
+
+The `defImage` function also supports [Buffer](https://nodejs.org/api/buffer.html)
+and [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob).
+
+
+This example takes a screenshot of bing.com and adds it to the images.
+
+```js
+const page = await host.browse("https://bing.com")
+const screenshot = await page.screenshot() // returns a node.js Buffer
+defImages(screenshot)
+```

packages/core/src/runpromptcontext.ts

@@ -235,31 +235,51 @@
     ) => {
         appendChild(node, createSchemaNode(name, schema, defOptions))
 
         return name
     }
 
-    const defImages = (files: StringLike, defOptions?: DefImagesOptions) => {
+    const defImages = (
+        files: ElementOrArray<string | WorkspaceFile | Buffer | Blob>,
+        defOptions?: DefImagesOptions
+    ) => {
         const { detail } = defOptions || {}
         if (Array.isArray(files))
             files.forEach((file) => defImages(file, defOptions))
         else if (typeof files === "string")
             appendChild(node, createImageNode({ url: files, detail }))
         else if (files instanceof Buffer) {
             const buffer: Buffer = files
             appendChild(
                 node,
                 createImageNode(
                     (async () => {
                         const mime = await fileTypeFromBuffer(buffer)
                         const b64 = await buffer.toString("base64")
                         const url = `data:${mime.mime};base64,${b64}`
                         return {
                             url,
                             detail,
                         }
                     })()
                 )
             )
+        } else if (files instanceof Blob) {
+            const blob: Blob = files
+            appendChild(
+                node,
+                createImageNode(
+                    (async () => {
+                        const buffer = Buffer.from(await blob.arrayBuffer())
+                        const mime = await fileTypeFromBuffer(buffer)
+                        const b64 = await buffer.toString("base64")
+                        const url = `data:${mime.mime};base64,${b64}`
+                        return {
+                            url,
+                            detail,
+                        }
+                    })()
+                )
+            )
         } else {
             const file: WorkspaceFile = files
             appendChild(

packages/core/src/types/prompt_template.d.ts

@@ -1,5 +1,7 @@
 type OptionsOrString<TOptions extends string> = (string & {}) | TOptions
 
+type ElementOrArray<T> = T | T[]
+
 interface PromptGenerationConsole {
     log(...data: any[]): void
     warn(...data: any[]): void
@@ -1537,7 +1539,7 @@
         options?: DefSchemaOptions
     ): string
     defImages(
-        files: StringLike | Buffer | Blob,
+        files: ElementOrArray<string | WorkspaceFile | Buffer | Blob>,
         options?: DefImagesOptions
     ): void
     defTool(

packages/core/src/types/prompt_type.d.ts

@@ -203,7 +203,7 @@ declare function defSchema(
  * @param options
  */
 declare function defImages(
-    files: StringLike | Buffer | Blob,
+    files: ElementOrArray<string | WorkspaceFile | Buffer | Blob>,    
     options?: DefImagesOptions
 ): void
 

packages/sample/genaisrc/azure-blog-storage.genai.mts

@@ -0,0 +1,35 @@
+import { BlobServiceClient } from "@azure/storage-blob"
+import { DefaultAzureCredential } from "@azure/identity"
+
+script({
+    parameters: {
+        account: {
+            description: "Azure Storage Account Name",
+            default: "myaccount",
+            type: "string",
+        },
+        container: {
+            description: "Azure Storage Container Name",
+            default: "mycontainer",
+            type: "string",
+        },
+    },
+})
+
+const { account, container } = env.vars
+const blobServiceClient = new BlobServiceClient(
+    `https://${account}.blob.core.windows.net`,
+    new DefaultAzureCredential()
+)
+const containerClient = blobServiceClient.getContainerClient(container)
+for await (const blob of containerClient.listBlobsFlat()) {
+    const blockBlobClient = containerClient.getBlockBlobClient(blob.name)
+    const downloadBlockBlobResponse = await blockBlobClient.download(0)
+    const body = await downloadBlockBlobResponse.readableStreamBody
+    const buffer = await body.read()
+    const res = await runPrompt((_) => {
+        _.defImages(buffer)
+        _.$`Describe the image.`
+    })
+    // do something with res?
+}

packages/sample/package.json

@@ -18,6 +18,8 @@
     "@agentic/calculator": "^7.0.0",
     "@agentic/core": "^7.0.0",
     "@agentic/weather": "^7.0.0",
+    "@azure/identity": "^4.4.1",
+    "@azure/storage-blob": "^12.24.0",
     "@tidyjs/tidy": "^2.5.2",
     "@xenova/transformers": "^2.17.2",
     "vectorstore": "^0.0.4",