update langchain

erniebot-agent/applications/erniebot_researcher/tools/preprocessing.py

@@ -2,22 +2,13 @@
 import os
 from typing import Any, List
 
-import faiss
 import jsonlines
 from langchain.docstore.document import Document
 from langchain.text_splitter import SpacyTextSplitter
 from langchain.vectorstores import FAISS
 from langchain_community.document_loaders import DirectoryLoader
-from llama_index import (
-    ServiceContext,
-    SimpleDirectoryReader,
-    StorageContext,
-    VectorStoreIndex,
-    load_index_from_storage,
-)
-from llama_index.node_parser import SentenceSplitter
+from llama_index import SimpleDirectoryReader
 from llama_index.schema import TextNode
-from llama_index.vector_stores.faiss import FaissVectorStore
 
 from erniebot_agent.memory import HumanMessage, Message
 from erniebot_agent.prompt import PromptTemplate
@@ -185,6 +176,7 @@ def build_index_langchain(
 
 def build_index_llama(index_name, embeddings, path=None, url_path=None, abstract=False, origin_data=None):
     # TODO: Adapt to llamaindex
+    pass
 
 
 def get_retriver_by_type(frame_type):

erniebot-agent/src/erniebot_agent/tools/llama_index_retrieval_tool.py

@@ -42,13 +42,5 @@ def __init__(
         self.threshold = threshold
 
     async def __call__(self, query: str, top_k: int = 3, filters: Optional[Dict[str, Any]] = None):
-        retriever = self.db.as_retriever(similarity_top_k=top_k)
-        nodes = retriever.retrieve(query)
-        docs = []
-        for doc in nodes:
-            if doc.score > self.threshold:
-                new_doc = {"content": doc.node.text, "score": doc.score}
-                if self.return_meta_data:
-                    new_doc["meta"] = doc.metadata
-                docs.append(new_doc)
-        return {"documents": docs}
+        # TODO: Adapt to llamaindex
+        pass