diff --git a/apps/hal9/app.py b/apps/hal9/app.py index e0418802..7c687510 100644 --- a/apps/hal9/app.py +++ b/apps/hal9/app.py @@ -13,6 +13,7 @@ from tools.image import create_image from tools.document import document_reply from tools.csv import csv_reply +from tools.image_analyzer import image_analyzer MODEL = "llama3-groq-70b-8192-tool-use-preview" def run(messages, tools): @@ -42,7 +43,8 @@ def run(messages, tools): build_streamlit, create_image, document_reply, - csv_reply + csv_reply, + image_analyzer ] tools = h9.describe(all_tools, model = "llama") @@ -51,7 +53,7 @@ def run(messages, tools): completion = run(messages, tools) h9.complete(completion, messages = messages, tools = all_tools, show = False, model = "llama") except Exception as e: - h9.event('error', e) + h9.event('error', str(e)) one_tool = h9.describe([generic_reply], model = "llama") completion = run(messages, one_tool) h9.complete(completion, messages = messages, tools = [generic_reply], show = False, model = "llama") diff --git a/apps/hal9/tools/image_analyzer.py b/apps/hal9/tools/image_analyzer.py new file mode 100644 index 00000000..79d3042e --- /dev/null +++ b/apps/hal9/tools/image_analyzer.py @@ -0,0 +1,33 @@ +import openai +import os + +def image_analyzer(image_url, prompt): + """Use this tool when the user provides a URL with an image extension such as JPG or PNG. +Parameters: + 'image_url' = URL containing the image or photograph. + 'prompt' = description of what the user wants to analyze in the image. If the user does not specify, it should default to "What's in this image?" +""" + client = openai.AzureOpenAI( + azure_endpoint = 'https://openai-hal9.openai.azure.com/', + api_key = os.environ['OPENAI_AZURE'], + api_version = '2024-02-15-preview', + ) + + response = client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url" : {"url": image_url,}, + }, + ], + } + ], + ) + + print(response.choices[0].message.content) + return response.choices[0].message.content \ No newline at end of file