modified: log_conf.json to INFO Level

CHANGELOG.md

@@ -5,6 +5,14 @@
     *Andrea Sponziello* 
 ### **Copyrigth**: *Tiledesk SRL*
 
+
+## [2024-05-03]
+
+### 0.1.12
+- fixed: send status 200 to webhook and id for status 400
+- added: DELETE of namespace by POST
+- fixed: /api/scrape/status check not only on redis but on Pinecone too
+
 ## [2024-05-02]
 
 ### 0.1.11

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tilellm"
-version = "0.1.11"
+version = "0.1.12"
 description = "tiledesk for RAG"
 authors = ["Gianluca Lorenzo <[email protected]>"]
 repository = "https://github.com/Tiledesk/tiledesk-llm"

tilellm/__main__.py

@@ -18,6 +18,7 @@
 from tilellm.models.item_model import (ItemSingle,
                                        QuestionAnswer,
                                        PineconeItemToDelete,
+                                       PineconeNamespaceToDelete,
                                        ScrapeStatusReq,
                                        ScrapeStatusResponse,
                                        PineconeIndexingResult)
@@ -115,6 +116,7 @@ async def reader(channel: aioredis.client.Redis):
                                                          ex=expiration_in_seconds)
 
                         logger.debug(f"Start {add_to_queue}")
+
                         raw_webhook = item.get('webhook', "")
                         if '?' in raw_webhook:
                             webhook, raw_token = raw_webhook.split('?')
@@ -126,6 +128,20 @@ async def reader(channel: aioredis.client.Redis):
 
                         logger.info(f"webhook: {webhook}, token: {token}")
 
+                        if webhook:
+                            res = PineconeIndexingResult(id=item.get('id'), status=200)
+                            try:
+                                async with aiohttp.ClientSession() as session:
+                                    res = await session.post(webhook,
+                                                             json=res.model_dump(exclude_none=True),
+                                                             headers={"Content-Type": "application/json",
+                                                                      "X-Auth-Token": token})
+                                    logger.info(res)
+                                    logger.info(f"200 ==>{await res.json()}")
+                            except Exception as ewh:
+                                logger.error(ewh)
+                                pass
+
                         pc_result = await add_pc_item(item_single)
                         # import datetime
                         # current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")
@@ -172,7 +188,7 @@ async def reader(channel: aioredis.client.Redis):
                 logger.error(f"Error {add_to_queue}")
                 import traceback
                 if webhook:
-                    res = PineconeIndexingResult(status=400, error=repr(e))
+                    res = PineconeIndexingResult(id=item.get('id'), status=400, error=repr(e))
                     async with aiohttp.ClientSession() as session:
                         response = await session.post(webhook,  json=res.model_dump(exclude_none=True),
                                                       headers={"Content-Type": "application/json", "X-Auth-Token": token})
@@ -278,14 +294,37 @@ async def list_namespace_items_main(namespace: str):
 @app.post("/api/scrape/status")
 async def scrape_status_main(scrape_status_req: ScrapeStatusReq,
                              redis_client: aioredis.client.Redis = Depends(get_redis_client)):
+    """
+    Check status of indexing
+    :param scrape_status_req:
+    :param redis_client:
+    :return:
+    """
     try:
         retrieved_data = await redis_client.get(f"{scrape_status_req.namespace}/{scrape_status_req.id}")
         if retrieved_data:
+            logger.debug(retrieved_data)
             scrape_status_response = ScrapeStatusResponse.model_validate(json.loads(retrieved_data.decode('utf-8')))
+            return JSONResponse(content=scrape_status_response.model_dump())
         else:
-            # FIXME recuperare da pinecone id e namespace...
-            raise Exception(f"Not Found, id: {scrape_status_req.id}, namespace:  {scrape_status_req.namespace}")
-        return JSONResponse(content=scrape_status_response.model_dump())
+            try:
+                retrieved_pinecone_data = await get_ids_namespace(metadata_id=scrape_status_req.id,
+                                                                  namespace=scrape_status_req.namespace)
+
+                if retrieved_pinecone_data.matches:
+                    logger.debug(retrieved_pinecone_data.matches[0].date)
+                    date_from_metadata = retrieved_pinecone_data.matches[0].date
+                    scrape_status_response = ScrapeStatusResponse(
+                        status_message=f"Indexing finished - verified in Pinecone metadata - date:{date_from_metadata}",
+                        status_code=3,
+                        queue_order=-1)
+                    return JSONResponse(content=scrape_status_response.model_dump())
+
+                else:
+                    raise Exception("Pinecone data not found")
+            except Exception as int_ex:
+                raise Exception(f"{repr(int_ex)}, id: {scrape_status_req.id}, namespace: {scrape_status_req.namespace}")
+
     except Exception as ex:
         raise HTTPException(status_code=400, detail=repr(ex))
 
@@ -304,6 +343,20 @@ async def delete_namespace_main(namespace: str):
         raise HTTPException(status_code=400, detail=repr(ex))
 
 
+@app.post("/api/delete/namespace")
+async def delete_namespace_main(namespace_to_delete: PineconeNamespaceToDelete):
+    """
+    Delete Pinecone namespace by namespace_id
+    :param namespace_to_delete:
+    :return:
+    """
+    try:
+        result = await delete_namespace(namespace_to_delete.namespace)
+        return JSONResponse(content={"success": "true", "message": f"{namespace_to_delete.namespace} is deleted from database"})
+    except Exception as ex:
+        raise HTTPException(status_code=400, detail={"success": "false", "message": repr(ex)})
+
+
 @app.delete("/api/id/{metadata_id}/namespace/{namespace}")
 async def delete_item_id_namespace_main(metadata_id: str, namespace: str):
     """

tilellm/models/item_model.py

@@ -19,6 +19,7 @@ class MetadataItem(BaseModel):
     source: str | None = None
     type: str | None = None
     embedding: str = Field(default_factory=lambda: "text-embedding-ada-002")
+    date: str = Field(default_factory=lambda: datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f"))
 
 
 class ChatEntry(BaseModel):
@@ -89,13 +90,18 @@ class PineconeQueryResult(BaseModel):
     metadata_id: str
     metadata_source: str
     metadata_type: str
+    date: Optional[str] = Field(default="Date not defined")
     text: Optional[str] | None = None
 
 
 class PineconeItems(BaseModel):
     matches: List[PineconeQueryResult]
 
 
+class PineconeNamespaceToDelete(BaseModel):
+    namespace: str
+
+
 class PineconeItemToDelete(BaseModel):
     id: str
     namespace: str

tilellm/store/pinecone_repository.py

@@ -142,7 +142,9 @@ async def delete_pc_ids_namespace(metadata_id: str, namespace: str):
     :return:
     """
     import pinecone
-    from langchain_community.vectorstores import Pinecone
+
+
+
 
     try:
         pc = pinecone.Pinecone(
@@ -236,6 +238,7 @@ async def get_pc_ids_namespace(metadata_id: str, namespace: str):
                                               metadata_id=obj.get('metadata').get('id'),
                                               metadata_source=obj.get('metadata').get('source'),
                                               metadata_type=obj.get('metadata').get('type'),
+                                              date=obj.get('metadata').get('date', 'Date not defined'),
                                               text=obj.get('metadata').get(const.PINECONE_TEXT_KEY)
                                               #su pod content, su Serverless text
                                               )
@@ -299,6 +302,7 @@ async def get_pc_all_obj_namespace(namespace: str):
                                               metadata_id=obj.get('metadata').get('id'),
                                               metadata_source=obj.get('metadata').get('source'),
                                               metadata_type=obj.get('metadata').get('type'),
+                                              date=obj.get('metadata').get('date', 'Date not defined'),
                                               text=None  # su pod content, su Serverless text
                                               )
                           )
@@ -396,6 +400,7 @@ async def get_pc_sources_namespace(source: str, namespace: str):
                                               metadata_id=obj.get('metadata').get('id'),
                                               metadata_source=obj.get('metadata').get('source'),
                                               metadata_type=obj.get('metadata').get('type'),
+                                              date=obj.get('metadata').get('date', 'Date not defined'),
                                               text=obj.get('metadata').get(const.PINECONE_TEXT_KEY)
                                               # su pod content, su Serverless text
                                               )
@@ -434,7 +439,6 @@ async def create_pc_index(embeddings, emb_dimension):
                                                     )  # text-key nuova versione è text
     else:
         logger.debug(f'Create index {const.PINECONE_INDEX} and embeddings ...')
-        # FIXME Cercare una soluzione megliore per gestire creazione di indici. Anche in produzione si potrebbe pensare di usare serverless...
 
         if os.environ.get("ENVIRON") == "dev":
             pc.create_index(const.PINECONE_INDEX,