[pull] master from Upsonic:master

gpt_computer_assistant/__init__.py

@@ -6,7 +6,7 @@
     from .tooler import Tool
 except:
     pass
-__version__ = '0.23.27'  # fmt: skip
+__version__ = '0.24.8'  # fmt: skip
 
 
 import os
@@ -159,4 +159,4 @@ def instance():
         the_instance = cloud_instance()
         the_instance.start()
 
-        return the_instance
+        return the_instancee

gpt_computer_assistant/agent/agent.py

@@ -4,14 +4,17 @@
     from ..llm_settings import llm_settings
     from ..tooler import *
     from ..display_tools import *
+    from ..cu.computer import *
     from ..teams import *
     from .agent_tools import get_tools
+
 except ImportError:
     from llm import get_model
     from utils.db import *
     from llm_settings import llm_settings
     from tooler import *
     from display_tools import *
+    from cu.computer import *
     from teams import *
     from agent.agent_tools import get_tools
 
@@ -58,34 +61,15 @@ def get_agent_executor():
         except ImportError:
             pass
 
-    if llm_settings[model]["provider"] == "openai":
-        tools += [
-            click_on_a_text_on_the_screen,
-            extract_possible_coordinates_of_text,
-            ocr_test,
-            click_on_a_icon_on_the_screen,
-            move_on_a_text_on_the_screen,
-            move_on_a_icon_on_the_screen,
-            mouse_scroll,
-        ]
-
-    if llm_settings[model]["provider"] == "azureai":
-        tools += [
-            click_on_a_text_on_the_screen,
-            extract_possible_coordinates_of_text,
-            ocr_test,
-            click_on_a_icon_on_the_screen,
-            move_on_a_text_on_the_screen,
-            move_on_a_icon_on_the_screen,
-            mouse_scroll,
-        ]
-
-    tools += [get_texts_on_the_screen]
+
+    tools += [computer_tool]
+
 
     if (
         llm_settings[model]["provider"] == "openai"
         or llm_settings[model]["provider"] == "groq"
-        or llm_settings[model]["provider"] == "azureai"
+        or llm_settings[model]["provider"] == "azureai",
+        llm_settings[model]["provider"] == "anthropic",
     ):
         return chat_agent_executor.create_tool_calling_executor(get_model(), tools)
 

gpt_computer_assistant/agent/agent_tools.py

@@ -49,16 +49,4 @@ def get_tiger_tools():
 
 
 def get_tools():
-    model = load_model_settings()
-
-    if not llm_settings[model]["tools"]:
-        return []
-
-    if is_online_tools_setting_active():
-        tools = get_tiger_tools()
-        if not tools:
-            tools = load_default_tools()
-    else:
-        tools = load_default_tools()
-
-    return tools
+    return []

gpt_computer_assistant/agent/assistant.py

@@ -69,6 +69,7 @@ def image_explaination():
             llm_settings[the_model]["provider"] == "openai"
             or llm_settings[the_model]["provider"] == "ollama"
             or llm_settings[the_model]["provider"] == "azureai"
+            or llm_settings[the_model]["provider"] == "anthropic"
         ):
             msg = get_agent_executor().invoke(
                 {"messages": llm_history + [the_message]}, config=config
@@ -144,7 +145,7 @@ def assistant(
             the_message.append(
                 {
                     "type": "image_url",
-                    "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"},
+                    "image_url": {"url": f"data:image/png;base64,{base64_image}"},
                 },
             )
         print("LEN OF IMAGE", len(base64_image))
@@ -161,14 +162,42 @@ def assistant(
         llm_settings[the_model]["provider"] == "openai"
         or llm_settings[the_model]["provider"] == "ollama"
         or llm_settings[the_model]["provider"] == "azureai"
+        or llm_settings[the_model]["provider"] == "anthropic"
     ):
+
+        if llm_settings[the_model]["provider"] == "anthropic":
+            # Filter all llm_history and if there is an "" in the content, Change it with "No response"
+            the_history = []
+            for message in llm_history:
+                if message.content == "" or message.content is None:
+                    if isinstance(message, SystemMessage):
+                        the_mes = SystemMessage(content="No response")
+                        the_history.append(the_mes)
+                    elif isinstance(message, HumanMessage):
+                        the_mes = HumanMessage(content="No response")
+                        the_history.append(the_mes)
+                    else:
+                        the_mes = AIMessage(content="No response")
+                        the_history.append(the_mes)
+                else:
+                    the_history.append(message)
+            llm_history = the_history
+
+
         if just_screenshot:
             msg = {"messages": llm_history + [the_message]}
             time.sleep(1)
         else:
+
             msg = get_agent_executor().invoke(
                 {"messages": llm_history + [the_message]}, config=config
             )
+
+
+
+
+
+
 
     if llm_settings[the_model]["provider"] == "google":
         the_history = []
@@ -242,8 +271,13 @@ def assistant(
         get_chat_message_history().clear()
         for message in currently_messages:
             get_chat_message_history().add_message(message)
+        if last_message == "":
+            last_message = "No response"
         get_chat_message_history().add_message(HumanMessage(content=[last_message]))
 
+
+    if the_last_messages[-1].content == "":
+        the_last_messages[-1].content = "No response"
     get_chat_message_history().add_message(the_last_messages[-1])
 
 
@@ -263,6 +297,12 @@ def assistant(
     print("The return", the_last_messages[-1].content)
 
     return_value = the_last_messages[-1].content
+    if isinstance(return_value, list):
+        the_text = ""
+        for each in return_value:
+            the_text += str(each)
+        return_value = the_text
+
     if return_value == "":
         return_value = "No response"
 

gpt_computer_assistant/cu/base.py

@@ -0,0 +1,69 @@
+from abc import ABCMeta, abstractmethod
+from dataclasses import dataclass, fields, replace
+from typing import Any
+
+from anthropic.types.beta import BetaToolUnionParam
+
+
+class BaseAnthropicTool(metaclass=ABCMeta):
+    """Abstract base class for Anthropic-defined tools."""
+
+    @abstractmethod
+    def __call__(self, **kwargs) -> Any:
+        """Executes the tool with the given arguments."""
+        ...
+
+    @abstractmethod
+    def to_params(
+        self,
+    ) -> BetaToolUnionParam:
+        raise NotImplementedError
+
+
+@dataclass(kw_only=True, frozen=True)
+class ToolResult:
+    """Represents the result of a tool execution."""
+
+    output: str | None = None
+    error: str | None = None
+    base64_image: str | None = None
+    system: str | None = None
+
+    def __bool__(self):
+        return any(getattr(self, field.name) for field in fields(self))
+
+    def __add__(self, other: "ToolResult"):
+        def combine_fields(
+            field: str | None, other_field: str | None, concatenate: bool = True
+        ):
+            if field and other_field:
+                if concatenate:
+                    return field + other_field
+                raise ValueError("Cannot combine tool results")
+            return field or other_field
+
+        return ToolResult(
+            output=combine_fields(self.output, other.output),
+            error=combine_fields(self.error, other.error),
+            base64_image=combine_fields(self.base64_image, other.base64_image, False),
+            system=combine_fields(self.system, other.system),
+        )
+
+    def replace(self, **kwargs):
+        """Returns a new ToolResult with the given fields replaced."""
+        return replace(self, **kwargs)
+
+
+class CLIResult(ToolResult):
+    """A ToolResult that can be rendered as a CLI output."""
+
+
+class ToolFailure(ToolResult):
+    """A ToolResult that represents a failure."""
+
+
+class ToolError(Exception):
+    """Raised when a tool encounters an error."""
+
+    def __init__(self, message):
+        self.message = message