application code update

crewai/tools/browser_tools.py

@@ -0,0 +1,39 @@
+import os
+import json
+import requests
+
+from crewai import Agent, Task
+from langchain.tools import tool
+from unstructured.partition.html import partition_html
+
+
+class BrowserTools():
+
+
+    @tool("Scrape website content")
+    def scrape_and_summarize_website(website):
+        """Useful for scrapping and summarizing a website content"""
+
+        url = f"https://chrome.browserless.io/content?token={os.environ['BROWSERLESS_API_KEY']}"
+        payload = json.dumps({"url": website})
+        headers = {"cache-control": "no-cache", "content-type": "application/json"}
+        response = requests.request("POST", url, headers=headers, data = payload)
+        elements = partition_html(text=response.text)
+        content = "\n\n".join([str(el) for el in elements])
+        content = [content[i:i + 8000] for i in range(0, len(content), 8000)]
+        summaries = []
+
+        for chunk in content:
+            agent = Agent(
+                role = "Principal Researcher",
+                goal="Do amazing research and summarize based on the content you are working with",
+                backstory="You are a Principal Researcher at a big company and you'll do research about a given topic.",
+                allow_delegation=False)
+            task = Task(
+                agent = agent,
+                description=f"Analyze and summarize the content below, make sure to include the most relevant information in the summary.
+                return only the summary nothing else. \n\nCONTENT\n------------\n{chunk}"
+            )
+            summary = task.execute()
+            summaries.append(summary)
+        return "\n\n".join(summaries)

langchain_local/agent.py

@@ -0,0 +1,51 @@
+import os
+
+from langchain_openai import ChatOpenAI
+from langchain_community.llms import ollama
+from langchain_community.chat_models import ChatOllama
+from langchain.prompts import ChatPromptTemplate
+from langchain_core.utils.function_calling import convert_to_openai_function
+from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
+from langchain.prompts import MessagesPlaceholder
+from langchain.schema.runnable import RunnablePassthrough
+from langchain.agents import AgentExecutor
+from langchain.memory import ConversationBufferMemory
+from langchain.agents.format_scratchpad import format_to_openai_functions
+from dotenv import load_dotenv, find_dotenv
+
+
+from tools.rag_tool import rag_one, rag_two
+from tools.serper_tool import serper_tool
+
+_ = load_dotenv(find_dotenv())
+
+api_key = os.getenv('OPENAI_API_KEY')
+
+tools = [rag_one, rag_two, serper_tool]
+
+functions = [convert_to_openai_function(f) for f in tools]
+model = ChatOpenAI(temperature= 0).bind_functions(functions=functions)
+#model = ChatOllama(model="mistral").bind(functions = functions)
+
+def generate(query: str):
+        """Function for interracting with the AI Agent"""
+
+        prompt = ChatPromptTemplate.from_messages([
+            ("system", "You are an expert agent with the ability to decide if a function is needed and route queries to the right function"),
+            MessagesPlaceholder(variable_name="chat_history"),
+            ("user", "{input}"),
+            MessagesPlaceholder(variable_name="agent_scratchpad")
+        ])
+
+        agent_chain = RunnablePassthrough.assign(
+                agent_scratchpad = lambda x: format_to_openai_functions(x["intermediate_steps"])
+        ) | prompt | model | OpenAIFunctionsAgentOutputParser()
+
+        memory = ConversationBufferMemory(return_message = True, memory_key = "chat_history")
+
+        agent_executor = AgentExecutor(agent = agent_chain, tools = tools, verbose= False, memory = memory)
+
+        result = agent_executor.invoke({"input": query})
+        return result['output']
+
+

langchain_local/tools/serper_tool.py

@@ -0,0 +1,18 @@
+import os
+import pprint
+
+from langchain_core.tools import Tool
+from langchain_community.utilities.google_serper import GoogleSerperAPIWrapper
+from dotenv import load_dotenv, find_dotenv
+
+_ = load_dotenv(find_dotenv())
+
+serper_api_key = os.getenv('SERPER_API_KEY')
+
+search = GoogleSerperAPIWrapper(serper_api_key = serper_api_key)
+
+serper_tool = Tool(
+    name = "Intermediate Answer",
+    func = search.run,
+    description="useful for when you need to ask with search"
+)

pyproject.toml

@@ -20,6 +20,7 @@ llama-index-llms-ollama = "^0.1.2"
 wikipedia = "^1.4.0"
 pypdf = "^4.1.0"
 docarray = "^0.40.0"
+unstructured = "^0.12.6"
 
 
 [tool.pyright]