fix: user-agent for scrape

Tiledesk · Jul 1, 2024 · b0cd3ce · b0cd3ce
1 parent 9c505d5
commit b0cd3ce
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,10 @@
     *Andrea Sponziello* 
 ### **Copyrigth**: *Tiledesk SRL*
 
+## [2024-07-01]
+### 0.2.5
+- fix: user-agent for scrape
+
 ## [2024-07-01]
 ### 0.2.4
 - fix: scrape_type=0

diff --git a/README.md b/README.md
@@ -87,6 +87,8 @@ pc.create_index(const.PINECONE_INDEX,
 ```
 
 ## Models
+Models for /api/ask
+
 ### OpenAI - engine: openai
 - gpt-3.5-turbo
 - gpt-4

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tilellm"
-version = "0.2.4"
+version = "0.2.5"
 description = "tiledesk for RAG"
 authors = ["Gianluca Lorenzo <[email protected]>"]
 repository = "https://github.com/Tiledesk/tiledesk-llm"
@@ -32,7 +32,7 @@ pinecone-client = "^4.1.1"
 python-dotenv = "^1.0.1"
 langchain_community = "0.2.x"
 tiktoken = "0.7.x"
-beautifulsoup4 ="^4.12.3"
+beautifulsoup4 = "^4.12.3"
 #uvicorn = "^0.28"
 unstructured= "0.14.x"
 #playwright = "^1.43.0"

diff --git a/tilellm/models/item_model.py b/tilellm/models/item_model.py
@@ -1,5 +1,5 @@
 from pydantic import BaseModel, Field,  field_validator, ValidationError
-from typing import Dict, Optional, List
+from typing import Dict, Optional, List, Union
 import datetime
 
 
@@ -76,9 +76,15 @@ def top_k_range(cls, v):
         return v
 
 
+class AWSAuthentication(BaseModel):
+    aws_access_key_id: str
+    aws_secret_access_key: str
+    region_name: str
+
+
 class QuestionToLLM(BaseModel):
     question: str
-    llm_key: str
+    llm_key: Union[str, AWSAuthentication]
     llm: str
     model: str = Field(default="gpt-3.5-turbo")
     temperature: float = Field(default=0.0)

diff --git a/tilellm/shared/utility.py b/tilellm/shared/utility.py
@@ -3,6 +3,7 @@
 
 import logging
 
+import langchain_aws
 from langchain_voyageai import VoyageAIEmbeddings
 from langchain_openai import OpenAIEmbeddings
 from tilellm.shared import const
@@ -119,6 +120,38 @@ async def wrapper(question, *args, **kwargs):
                                   max_tokens=question.max_tokens
                                   )
 
+        elif question.llm == "aws":
+            import os
+
+            os.environ["AWS_SECRET_ACCESS_KEY"] = question.llm_key.aws_secret_access_key
+            os.environ["AWS_ACCESS_KEY_ID"] = question.llm_key.aws_access_key_id
+
+            # chat_model = ChatBedrock(model_id=question.model,
+            #                         model_kwargs={"temperature": question.temperature,"max_tokens":question.max_tokens },
+            #                         region_name="eu-central-1"
+            #                         )
+
+            import boto3
+            #session = boto3.Session(
+            #                        aws_access_key_id=question.llm_key.aws_secret_access_key,
+            #                        aws_secret_access_key=question.llm_key.aws_secret_access_key,
+            #                        region_name=question.llm_key.region_name
+            #                        )
+
+
+
+            chat_model = ChatBedrockConverse(
+                model=question.model,
+                temperature=question.temperature,
+                max_tokens=question.max_tokens,
+                region_name=question.llm_key.region_name
+
+                #                                 base_url="http://bedroc-proxy-paacejvmzcgv-121947512.eu-central-1.elb.amazonaws.com/api/v1/",
+
+            )  # model_kwargs={"temperature": 0.001},
+
+            #print(chat_model.session)
+
         else:
             chat_model = ChatOpenAI(api_key=question.llm_key,
                                     model=question.model,

diff --git a/tilellm/tools/document_tool_simple.py b/tilellm/tools/document_tool_simple.py
@@ -15,11 +15,13 @@ def get_content_by_url(url: str, scrape_type: int):
         urls = [url]
         if scrape_type == 0:
             loader = UnstructuredURLLoader(
-                urls=urls, mode="elements", strategy="fast", continue_on_failure=False
+                urls=urls, mode="elements", strategy="fast", continue_on_failure=False,
+                headers={'user-agent': 'Mozilla/5.0'}
             )
         else:
             loader = UnstructuredURLLoader(
-                urls=urls, mode="single", continue_on_failure=False
+                urls=urls, mode="single", continue_on_failure=False,
+                headers={'user-agent': 'Mozilla/5.0'}
             )
         docs = loader.load()