utils.ai.AIAgent,CallTool,CallToolOutput (minor) new components for d…

…efining AI agents

src/appmixer/utils/ai/AIAgent/AIAgent.js

@@ -0,0 +1,226 @@
+'use strict';
+
+const OpenAI = require('openai');
+
+module.exports = {
+
+    start: async function(context) {
+
+        const assistant = await this.createAssistant(context);
+        return context.stateSet('assistant', assistant);
+    },
+
+    createAssistant: async function(context) {
+
+        const flowDescriptor = context.flowDescriptor;
+        const agentComponentId = context.componentId;
+        const toolsPort = 'tools';
+
+        // Create a new assistant with tools defined in the branches connected to my 'tools' output port.
+        const tools = {};
+        let error;
+
+        // Find all components connected to my 'tools' output port.
+        Object.keys(flowDescriptor).forEach((componentId) => {
+            const component = flowDescriptor[componentId];
+            const sources = component.source;
+            Object.keys(sources || {}).forEach((inPort) => {
+                const source = sources[inPort];
+                if (source[agentComponentId] && source[agentComponentId].includes(toolsPort)) {
+                    tools[componentId] = component;
+                    // assert(flowDescriptor[componentId].type === 'appmixer.utils.ai.CallTool')
+                    if (component.type !== 'appmixer.utils.ai.CallTool') {
+                        error = `Component ${componentId} is not of type 'ai.CallTool' but ${comopnent.type}.
+                            Every tool chain connected to the '${toolsPort}' port of the AI Agent
+                            must start with 'ai.CallTool' and end with 'ai.CallToolOutput'.
+                            This is where you describe what the tool does and what parameters should the AI model provide to it.`;
+                    }
+                }
+            });
+        });
+
+        // Teach the user via logs that they need to use the 'ai.CallTool' component.
+        if (error) {
+            throw new context.CancelError(error);
+        }
+
+        const toolsDefinition = this.getToolsDefinition(tools);
+
+        const instructions = context.properties.instructions || null;
+        await context.log({ step: 'create-assistant', tools: toolsDefinition, instructions });
+
+        const apiKey = context.config.apiKey;
+
+        if (!apiKey) {
+            throw new context.CancelError('Missing \'apiKey\' system setting of the appmixer.utils.ai module pointing to OpenAI. Please provide it in the Connector Configuration section of the Appmixer Backoffice.');
+        }
+
+        const client = new OpenAI({ apiKey });
+        const assistant = await client.beta.assistants.create({
+            model: context.config.AIAgentModel || 'gpt-4o',
+            instructions,
+            tools: toolsDefinition
+        });
+
+        await context.log({ step: 'created-assistant', assistant });
+        return assistant;
+    },
+
+    getToolsDefinition: function(tools) {
+
+        // https://platform.openai.com/docs/assistants/tools/function-calling
+        const toolsDefinition = [];
+
+        Object.keys(tools).forEach((componentId) => {
+            const component = tools[componentId];
+            const toolParameters = {
+                type: 'object',
+                properties: {}
+            };
+            component.config.properties.parameters.ADD.forEach((parameter) => {
+                toolParameters.properties[parameter.name] = {
+                    type: parameter.type,
+                    description: parameter.description
+                };
+            });
+            const toolDefinition = {
+                type: 'function',
+                function: {
+                    name: componentId,
+                    description: component.config.properties.description,
+                    parameters: toolParameters
+                }
+            };
+            toolsDefinition.push(toolDefinition);
+        });
+        return toolsDefinition;        
+    },
+
+    handleRunStatus: async function(context, client, thread, run) {
+
+        await context.log({ step: 'run-status', run });
+
+        // Check if the run is completed
+        if (run.status === 'completed') {
+            let messages = await client.beta.threads.messages.list(thread.id);
+            await context.log({ step: 'completed-run', run, messages });
+            await context.sendJson({
+                answer: messages.data[0].content[0].text.value,
+                prompt: context.messages.in.content.prompt
+            }, 'out');
+        } else if (run.status === 'requires_action') {
+            await this.handleRequiresAction(context, client, thread, run);
+        } else {
+            await context.log({ step: 'unexpected-run-state', run });
+        }
+    },
+
+    handleRequiresAction: async function(context, client, thread, run) {    
+
+        await context.log({ step: 'requires-action', run });
+
+        // Check if there are tools that require outputs.
+        if (
+            run.required_action &&
+            run.required_action.submit_tool_outputs &&
+            run.required_action.submit_tool_outputs.tool_calls
+        ) {
+            // Loop through each tool in the required action section.
+            const toolOutputs = [];
+            const toolCalls = [];
+            for (const toolCall of run.required_action.submit_tool_outputs.tool_calls) {
+                const componentId = toolCall.function.name;
+                const args = JSON.parse(toolCall.function.arguments);
+                toolCalls.push({ componentId, args, toolCallId: toolCall.id });
+
+                await context.log({ step: 'call-tool', toolCallId: toolCall.id, componentId, args });
+                await context.callAppmixer({
+                    endPoint: `/flows/${context.flowId}/components/${componentId}`,
+                    method: 'POST',
+                    body: args,
+                    qs: { enqueueOnly: true, correlationId: toolCall.id }
+                });
+            }
+
+            // Output of each tool is expected to be stored in the service state
+            // under the ID of the tool call. This is done in the CallToolOutput component.
+            // Collect outputs of all the required tool calls.
+            await context.log({ step: 'collect-tools-output', threadId: thread.id, runId: run.id });
+            const outputs = [];
+            const pollInterval = 1000;
+            const pollTimeout = 20000;
+            const pollStart = Date.now();
+            while ((outputs.length < toolCalls.length) && (Date.now() - pollStart < pollTimeout)) {
+                for (const toolCall of toolCalls) {
+                    const output = await context.service.stateGet(toolCall.toolCallId);
+                    if (output) {
+                        outputs.push({ tool_call_id: toolCall.toolCallId, output });
+                        await context.service.stateUnset(toolCall.toolCallId);
+                    }
+                }
+                // Sleep.
+                await new Promise((resolve) => setTimeout(resolve, pollInterval));
+            }
+            await context.log({ step: 'collected-tools-output', threadId: thread.id, runId: run.id, outputs });
+
+            // Submit tool outputs to the assistant.            
+            if (outputs && outputs.length) {
+                await context.log({ step: 'tool-outputs', tools: toolCalls, outputs });
+                run = await client.beta.threads.runs.submitToolOutputsAndPoll(
+                    thread.id,
+                    run.id,
+                    { tool_outputs: outputs },
+                );
+            } else {
+                await context.log({ step: 'no-tool-outputs', tools: toolCalls });
+            }
+
+            // Check status after submitting tool outputs.
+            return this.handleRunStatus(context, client, thread, run);
+        }
+    },
+
+    receive: async function(context) {
+
+        const { prompt } = context.messages.in.content;
+        let threadId = context.messages.in.content.threadId || context.messages.in.correlationId;
+        const apiKey = context.config.apiKey;
+        const client = new OpenAI({ apiKey });
+        const assistant = await context.stateGet('assistant');
+
+        // Check if a thread with a given ID exists.
+        let thread;
+        if (threadId) {
+            thread = await context.stateGet(threadId);
+        }
+        if (!thread) {
+            await context.log({ step: 'create-thread', assistantId: assistant.id, internalThreadId: threadId });
+            thread = await client.beta.threads.create();
+            await context.stateSet(threadId, thread);
+        } else {
+            await context.log({ step: 'use-thread', assistantId: assistant.id, thread });
+        }
+
+        await context.log({ step: 'create-thread-message', assistantId: assistant.id, threadId: thread.id });
+        await client.beta.threads.messages.create(thread.id, {
+            role: 'user',
+            content: prompt
+        });
+
+        await context.log({ step: 'create-thread-run', assistantId: assistant.id, threadId: thread.id });
+        const run = await client.beta.threads.runs.createAndPoll(thread.id, {
+            assistant_id: assistant.id
+        });
+
+        await context.log({ step: 'created-thread-run', assistantId: assistant.id, threadId: thread.id, runId: run.id });
+        await this.handleRunStatus(context, client, thread, run);
+    },
+
+    stop: async function(context) {
+
+        const apiKey = context.config.apiKey;
+        const client = new OpenAI({ apiKey });
+        const assistant = await context.stateGet('assistant');
+        await client.beta.assistants.del(assistant.id);
+    }
+};

src/appmixer/utils/ai/AIAgent/component.json

@@ -0,0 +1,69 @@
+{
+    "name": "appmixer.utils.ai.AIAgent",
+    "author": "Appmixer <[email protected]>",
+    "description": "Build an AI agent responding with contextual answers or performing contextual actions.",
+    "properties": {
+        "schema": {
+            "type": "object",
+            "properties": {
+                "instructions": { "type": "string", "maxLength": 256000 }
+            }
+        },
+        "inspector": {
+            "inputs": {
+                "instructions": {
+                    "type": "textarea",
+                    "label": "Instructions",
+                    "index": 1,
+                    "tooltip": "The system instructions that the assistant uses. The maximum length is 256,000 characters. For example 'You are a personal math tutor.'."
+                }
+            }
+        }
+    },
+    "inPorts": [{
+        "name": "in",
+        "schema": {
+            "type": "object",
+            "properties": {
+                "prompt": { "type": "string" },
+                "threadId": { "type": "string" }
+            },
+            "required": ["prompt"]
+        },
+        "inspector": {
+            "inputs": {
+                "prompt": {
+                    "label": "Prompt",
+                    "type": "textarea",
+                    "index": 1
+                },
+                "threadId": {
+                    "label": "Thread ID",
+                    "type": "text",
+                    "index": 2,
+                    "tooltip": "By setting a thread ID you can keep the context of the conversation."
+                }
+            }
+        }
+    }],
+    "outPorts": [{
+        "name": "out",
+        "options": [{
+            "label": "Answer",
+            "value": "answer",
+            "schema": { "type": "string" }
+        }, {
+            "label": "Prompt",
+            "value": "prompt",
+            "schema": { "type": "string" }
+        }]
+    }, {
+        "name": "tools",
+        "options": [{
+            "label": "Prompt",
+            "value": "prompt",
+            "schema": { "type": "string" }
+        }]
+    }],
+    "icon": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGUAAABiCAYAAABJeR13AAABDWlDQ1BJQ0MgUHJvZmlsZQAAKJFjYGCSSCwoyGESYGDIzSspCnJ3UoiIjFJgf8HAwcDJwMegyaCWmFxc4BgQ4MMABDAaFXy7xsAIoi/rgszClMcLuFJSi5OB9B8gTkkuKCphYGBMALKVy0sKQOwWIFskKRvMngFiFwEdCGSvAbHTIewDYDUQ9hWwmpAgZyD7BZDNlwRh/wCx08FsJg4QG2ov2A2OQHenKgB9T6LjCYGS1IoSEO2cX1BZlJmeUaIAsckzL1lPR8HIwMiEgQEU3hDVnwPB4cgodgYhhgAIscq9wJgIYmBg2YkQCwP6b40+A4MsM0JMTYmBQaiegWFjQXJpURnUGEbGswwMhPgAhBpLo/XijBMAAABWZVhJZk1NACoAAAAIAAGHaQAEAAAAAQAAABoAAAAAAAOShgAHAAAAEgAAAESgAgAEAAAAAQAAAGWgAwAEAAAAAQAAAGIAAAAAQVNDSUkAAABTY3JlZW5zaG90r1CPNgAAAdVpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDYuMC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iPgogICAgICAgICA8ZXhpZjpQaXhlbFlEaW1lbnNpb24+OTg8L2V4aWY6UGl4ZWxZRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+MTAxPC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+CsiCPvwAAAz7SURBVHgB7V1XqBU9EI69996wi72+2MVeUewNRB/UJx9UFDsootgVRLFgQURF7KJYQFGx96547Vix97r/fPnJkt3NtnP37NkrO3DvbpJJMpnZSZlMcjJpBCyGSHEgc6SoiYnhHIiFEsEPIXSh/Pr1i129epV9//7dNztOnTrF0tLSfOdDHuT1C6ARtILmUAFjSljw9+9frUWLFhjDtDp16mjUaM9VT5o0iefLli2bdvz4cc/5jh07piEP6pw8ebLnfKANNCIfaAbtYQELqyLU8/TpU95INBR/Fy5c8Fx9/fr19bxTp071nA+4oj6U4RVAm8iH55MnT7xmTTdeqN2XuRswh6nxtkBfqp4mv+uRNi8yrvxug65Hm2kzh3XEJLyEKpQk0P9PFhkLJYJijYUSCyWCHIggSbGmxEKJIAciSFKsKbFQIsiBCJKUNSyaHj58yA4ePGiobtmyZWzHjh2GOLvAixcv9KQjR46wCRMm6GGnFzLJ6Mkow2u+58+f6/nwcvjwYdahQwdWvnx5Q3xSAum2CTgU8PnzZ23BggVajRo1DCYLakiGDdeqVUtbuHCh9uXLF4eWpy8pabYv0gCtVKlSGZb5bh9OmTJltF27dqWP+za5MyE+aBWcMmUKmzVrFgRuKJqstaxcuXIsV65chvioB759+8bIIGkx4WfKlImhrTNmzAi2CTbCSjiaiLRoR+fOnbW9e/dqX79+TbjcVGcE7WhDp06dLO3zY7X20o5Au6+dO3dq9PXoRBcuXDhpKu6lccnCQTsLFSqktxNtDrIrC0woGNRLly6tEwqB0K5dsviS8nKvXLliEAzGmKAG/8CEglkWdaz6X5BfTsolYEMANEZu86JFi2ww/UUHNtDTVJHdvHmTD3g0hrB9+/Y5Dn4fPnxgWLuQhjnipSoxb968rEKFCqxAgQKOJNAYww4cOMBxwIPr16874ntK9CdDNTYx1/DFYEC0A2yzYrDMmjWrIQ8RG7kwaMQk5eLFi3bN0Xbv3m2gG7xIL9hqChXMaE/dMg1USRqr3ZEjR/IkTHuhBapp76ZNm9jQoUPZz58/VcVENi5Hjhxs7dq1bODAgRYaMV2GNont4hUrVrB27dpZ8MwR2bNnZzQOMUyrLaCSKjFNa968ueELoIyewpUqVVIVqZ0/f14jQjyV4bWuMPFIMLYaU7FixYTaBS8Z8NoMStsXxoYTJ05YBOglIk+ePEo0LLKEhmTJkoXboPDlqTRKWUDIkbQuYdDsOXPmsD9//rAfP34wclFSjpUYfxIB2OVu3brF6tata8xulhLCpJK6zxNh+/oKqAJLke/fvzeMIX78ryyFhRwxceJEvf3wH6Ou2UIB2uyXT8CHXxl4bQalpuTMmZOdO3eO3b59m/3+/dsoRUUI1t41a9YoUv6PwixLLkfVN9tmTnHCoEGD2OzZszkVGDfQFsuXLdE4ZMgQNmLECClG/YpxioTC8DSDUihAAnK9evXM+MowGR6V8SISg6EMdl2cjBOVd3PXhG7NCWDba9asmROKa1q88+jKovARYqGEz3PXGmOhuLIofIRYKOHz3LXGWCiuLAofIRZK+Dx3rdF2SuyaM0UIsDQULFiQ1a5d2xMFWInDkoB1EtYZYnVOCzaWP39+Rvs+avuTp9KTg5ShNAWLMrIX8fXTypUrHTkyZswYVrRoUYaFsGB+iRIlGG3EMbJVMbLR8fRixYopTSeOhSc5McMIBfszq1at4uzA4Z/Ro0dzZwYVf+7du8dow4m9efNGlWyIA87cuXMNcakO2HZfMMRt2bJFNyI6Efrq1Sun5EDSzB4jWFkvWbKEzZ8/31J+kSJFuDkdWwheoG3btl7QPOFAg2lH0hUXpvsBAwaw8ePHW3HNxjCEyeaVkIGNStdUBkk6mWso78GDB6pqbeMuX75syI968EfdkUbjhDIfeThqpF3a0aNHNRqHtKVLlxrKwGFWxF+6dEmZX0SCVlEfnmiLGRI1SMLh4s6dO+biNKWmYPCDfYocAYiO1MPq1auVRLx8+ZJvxXbt2tWSXrJkSYZtaQHmI+INGjTgbRTpqXjmzp2bkVeMpWqlUDD40VFnRs4PfB/BkssUgX2BkydPmmKDCWLWhG5UABh96NAh3eq8efNmphKKwBdPzMJkUFln5fRE35s2bconI275UX+PHj0YeG0GpVCA1LBhQ/5nzqAKYwMrWUKBM/fr16/1akeNGsWntcJZfM+ePfzDcWOyrCnYZKP9d73MIF9at27NZs6cma4iIz/72rp1q95ADODY/8YAKQCDOXwE3EDePsA0OcoQaaFgoSfPZKDucMzo2bOnYXNo27ZtrjyOheLKIm8IGNfkrqt///48I1b0OCsiAGOf8CYRceanLBS3rs6cN+xwpDVl+/btOj8wILZp00YP9+nTR39/+/Ytw9jjBLJQ4u7LiVMOaTR5N3RdvXr1MgzO3bt399WFyUKJNcWB8U5JZ86c4c6AAqdfv37ilT/Rhcmag7EHY5AdyEKJqluToD0580JRejqecteFYjDNFF4lotj79++LVwZTD+xdY8eOVVp9ZYeHWCg62/y9mIXiNmag9HHjxnGLsMrFR9aUeEzxJwuOTbauhG7AQ2bzqV5RvawpMG9EGWy7L/KO52YW4Wrq1Aj5WLQTntc0WUuwYHQyo+AYtljdo/zBgwcrq5E1JZl+Z16Pk2OygclKo0aNLPQqhYK1QatWrVJmkJQXg3379mXLly+3EC4iMJbA+IjZGoAcyVmVKlVEsv6UhZJMTYG5yavJCdsOD8nj0mz/Uq5T3r17x2R111sWwgtcZcXhI1SHr8kJihcvznBYR4DQGggJAkNb8Pfx40eB4noQSEdM8gs+FNBmBqWmVKtWjXubw+vcS/eFxssrb3MlfsJy15UvXz4GA58bAEecoIIdDAJo0qSJQbhyGZilQftwNgRnRNavX89g3Q0C8NXjQ3EDbHLBpxq8toBlhyWBCHjRU8H8L72bXC1bttTL6t27tydqSJB6HtBBH5MhLGizezrV43eTK4gTBcrui4hPGdSsWZPXjcEYzg9eAJZjOEMAoCHdunVTDqB2ZakGWzvcMOKV3VcYFdvVgWMVOE6AQ6Bu3vyiDHRzN27cYPRV8+4AeyVnz57lXRoMlcK1CF0xVv14wvkCG1/weMGkJkoQOaGgn8fX7hewShdahryZM2d2PEfit/ww8SPXfYXZ+KjWFQslgpKJhRILJYIciCBJsabEQokgByJIku2UGLava9eu6U5vTrTj1jgnMG8qRfWSHFUbzLS6GTOxdQALuxtg6l+9enWmLE9lxyAjmUaLN1+mCiKC46vMLLgQQL4gBxcGZBSg21l1PgR9uQGuD8HFD2ZQagokDZNyIkAVWLLhfEj79u3Z/v37eZo4eoCLA5RfiqWE8CPQU2zcuJHNmzdPrxxtQFvMoGqzGUcVhgUCvJZ9DTieWUoI45Y7uthA/0II0fO73YU5uJ6JNnY8l+OnzjBwQbudh36iF+bgl49Ut+nZXi0F2xCulpKv7+BSVPzzerUUHLVxtZTs16soLnJR2CVct26dwV1WEAmNwtVSgk842GT58gWy9IQ/M26ngDnIAipN8RtnNm87XcIGjenSpYv+42VEUGS1B2MIbUXbagj4ZL6E7dGjR37ZZ8G31RSL9Fwi5OsKcYWfGD/ssn369IlbdfGMIsDyjO0APJ1Avq4Qh2MxY003WMSUYASuGidi9D9cevmvg3lzbfHixYE0ObDbVjFg4RpYIRjc24trYv9VwJE/8tLU21u2bFnloJ1I+wMTCirHtbc4xycL5l/UGGiILBC0GWNLUBCoUECU/GOXQjgdO3bkRKumf0E1JNnlgHYwno5g6B+daN+0adMCrT6wgZ4I1IGI5L6/RKkehxds02IaGNUFo4FYKYBpL0xJYtorkmAqoY+QTZ8+XUQF8wxUxFJh6MrkMYaotXxhGTkObQuyy5JYpwXefcmFQ+Vx1ThNFf8ZgdDUn7cpmV1xUrovlQ4/fvyY+/wOHz5cT4bXilePFVwcKhz+vB6LRkU4oid+9hxOcsOGDdPrd3p59uwZ27Bhg46ClTpsXxn+559krcE7nScxaMzp06fNKLZh8TPkxCVfP2EufjId+VQWbLsK03tLhl25XuIVhhf944hfUsSBWCgpYrxTtbFQnLiTorRYKClivFO1sVCcuJOitFgoKWK8U7WxUJy4k6K0WCgpYrxTtaEKBTcQyYAjZl5B3svG/rZXSDSfmVZz2Gv9ieCFKhScL6Tjc5xO/AyIfIDUjXhxbBvMkW8wcssHXMFQUYZbHqRjaxe/bwIAzaA9LAjN9iUaBPM3TgBXrVrVcOGNSHd6woYF+1XlypWd0CxpaWlp3G7WuHFjS5pTBLxu7t69yw8jJesmPVX9oQtFRUQcZ+RAqN2Xseo4ZMeBWCh2nElh/H/liwvAjmbZQwAAAABJRU5ErkJggg=="
+}

src/appmixer/utils/ai/CallTool/CallTool.js

@@ -0,0 +1,26 @@
+'use strict';
+
+module.exports = {
+
+    receive: async function(context) {
+
+        if (context.properties.generateOutputPortOptions) {
+            return this.getOutputPortOptions(context);
+        }
+
+        if (context.messages.webhook) {
+            // Tool chain triggered by AI Agent.
+            await context.sendJson(context.messages.webhook.content.data, 'out');
+            return context.response({});
+        }
+    },
+
+    getOutputPortOptions(context) {
+
+        const options = [];
+        context.properties.parameters.ADD.forEach(parameter => {
+            options.push({ label: parameter.name, value: parameter.name, schema: { type: parameter.type } });
+        });
+        return context.sendJson(options, 'out');
+    }
+};