modified default value for scarpe type:4

Tiledesk · Sep 14, 2024 · c36c186 · c36c186
1 parent 8509035
commit c36c186
Show file tree

Hide file tree

Showing 6 changed files with 13 additions and 5 deletions.
diff --git a/.gitignore b/.gitignore
@@ -113,6 +113,9 @@ celerybeat.pid
 .venv
 .environ
 .environ.prod
+deploy_old.sh
+docker-compose-test.yml
+docker-compose.yml
 env/
 venv/
 ENV/

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,11 @@
     *Andrea Sponziello* 
 ### **Copyrigth**: *Tiledesk SRL*
 
+## [2024-09-14]
+### 0.2.18
+- upgrade: worker
+- modify: default value for scrape type: 4
+
 ## [2024-09-05]
 ### 0.2.17
 - fix: nltk download on Dockerfile   

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tilellm"
-version = "0.2.17"
+version = "0.2.18"
 description = "tiledesk for RAG"
 authors = ["Gianluca Lorenzo <[email protected]>"]
 repository = "https://github.com/Tiledesk/tiledesk-llm"

diff --git a/tilellm/models/item_model.py b/tilellm/models/item_model.py
@@ -10,8 +10,8 @@ class ParametersScrapeType4(BaseModel):
     tags_to_extract: Optional[List[str]] = Field(default_factory=list)
     unwanted_classnames: Optional[List[str]] = Field(default_factory=list)
     desired_classnames: Optional[List[str]] = Field(default_factory=list)
-    remove_lines: Optional[bool] = Field(default=False)
-    remove_comments: Optional[bool] = Field(default=False)
+    remove_lines: Optional[bool] = Field(default=True)
+    remove_comments: Optional[bool] = Field(default=True)
 
     @model_validator(mode='after')
     def check_booleans(cls, values):

diff --git a/tilellm/store/pinecone/pinecone_repository_serverless.py b/tilellm/store/pinecone/pinecone_repository_serverless.py
@@ -135,7 +135,7 @@ async def add_pc_item(self, item, embedding_obj=None, embedding_dimension=None):
 
             else:
                 metadata = MetadataItem(id=metadata_id, source=source, type=type_source, embedding=embedding)
-                document = Document(page_content=content, metadata=metadata.dict())
+                document = Document(page_content=content, metadata=metadata.model_dump()) #tolto dict()
 
                 chunks.extend(self.chunk_data(data=[document], chunk_size=chunk_size, chunk_overlap=chunk_overlap))
                 total_tokens, cost = self.calc_embedding_cost(chunks, embedding)

diff --git a/worker/package.json b/worker/package.json
@@ -9,7 +9,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@tiledesk/tiledesk-train-jobworker": "^0.0.23",
+    "@tiledesk/tiledesk-train-jobworker": "^0.0.24",
     "dotenv": "^16.4.5",
     "express": "^4.19.2"
   }