From 403b08f3cf534d7bbda05d71d8405acdddbd4e9c Mon Sep 17 00:00:00 2001 From: cjkindel Date: Fri, 12 Jul 2024 13:57:54 -0700 Subject: [PATCH 01/10] add default count for qdrant vsd --- griptape/drivers/vector/qdrant_vector_store_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/griptape/drivers/vector/qdrant_vector_store_driver.py b/griptape/drivers/vector/qdrant_vector_store_driver.py index 34345b6df..d3038839e 100644 --- a/griptape/drivers/vector/qdrant_vector_store_driver.py +++ b/griptape/drivers/vector/qdrant_vector_store_driver.py @@ -87,7 +87,7 @@ def delete_vector(self, vector_id: str) -> None: def query( self, query: str, - count: Optional[int] = None, + count: Optional[int] = BaseVectorStoreDriver.DEFAULT_QUERY_COUNT, namespace: Optional[str] = None, include_vectors: bool = False, **kwargs, From b81441b4bb90f84fc7e4adae3ac5b522a7e0f814 Mon Sep 17 00:00:00 2001 From: cjkindel Date: Fri, 12 Jul 2024 13:58:11 -0700 Subject: [PATCH 02/10] standardize pinecone snippet --- .../drivers/vector-store-drivers.md | 46 +++++-------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/docs/griptape-framework/drivers/vector-store-drivers.md b/docs/griptape-framework/drivers/vector-store-drivers.md index ea2b72a56..e673c15e1 100644 --- a/docs/griptape-framework/drivers/vector-store-drivers.md +++ b/docs/griptape-framework/drivers/vector-store-drivers.md @@ -49,7 +49,6 @@ results = vector_store_driver.query( values = [r.to_artifact().value for r in results] print("\n\n".join(values)) - ``` ### Griptape Cloud Knowledge Base @@ -73,7 +72,6 @@ results =vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] print("\n\n".join(values)) - ``` ### Pinecone @@ -86,49 +84,27 @@ The [PineconeVectorStoreDriver](../../reference/griptape/drivers/vector/pinecone Here is an example of how the Driver can be used to load and query information in a Pinecone cluster: ```python -import os -import hashlib -import json -from urllib.request import urlopen +import os +from griptape.artifacts import BaseArtifact from griptape.drivers import PineconeVectorStoreDriver, OpenAiEmbeddingDriver +from griptape.loaders import WebLoader -def load_data(driver: PineconeVectorStoreDriver) -> None: - response = urlopen( - "https://raw.githubusercontent.com/wedeploy-examples/" - "supermarket-web-example/master/products.json" - ) - - for product in json.loads(response.read()): - driver.upsert_text( - product["description"], - vector_id=hashlib.md5(product["title"].encode()).hexdigest(), - meta={ - "title": product["title"], - "description": product["description"], - "type": product["type"], - "price": product["price"], - "rating": product["rating"], - }, - namespace="supermarket-products", - ) # Initialize an Embedding Driver embedding_driver = OpenAiEmbeddingDriver(api_key=os.environ["OPENAI_API_KEY"]) -vector_store_driver = PineconeVectorStoreDriver( - api_key=os.environ["PINECONE_API_KEY"], - environment=os.environ["PINECONE_ENVIRONMENT"], - index_name=os.environ['PINECONE_INDEX_NAME'], - embedding_driver=embedding_driver, -) +vector_store_driver = PineconeVectorStoreDriver(embedding_driver=embedding_driver) -load_data(vector_store_driver) +# Load Artifacts from the web +artifacts = WebLoader(max_tokens=100).load("https://www.griptape.ai") + +# Upsert Artifacts into the Vector Store Driver +[vector_store_driver.upsert_text_artifact(a, namespace="griptape") for a in artifacts] results = vector_store_driver.query( - "fruit", + "creativity", count=3, - filter={"price": {"$lte": 15}, "rating": {"$gte": 4}}, - namespace="supermarket-products", + namespace="griptape" ) values = [r.to_artifact().value for r in results] From e8489bd2f148baf2e5052516cefc49073efa2173 Mon Sep 17 00:00:00 2001 From: cjkindel Date: Fri, 12 Jul 2024 13:59:24 -0700 Subject: [PATCH 03/10] fix pinecone snippet --- docs/griptape-framework/drivers/vector-store-drivers.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/griptape-framework/drivers/vector-store-drivers.md b/docs/griptape-framework/drivers/vector-store-drivers.md index e673c15e1..c86bb6a5c 100644 --- a/docs/griptape-framework/drivers/vector-store-drivers.md +++ b/docs/griptape-framework/drivers/vector-store-drivers.md @@ -93,7 +93,12 @@ from griptape.loaders import WebLoader # Initialize an Embedding Driver embedding_driver = OpenAiEmbeddingDriver(api_key=os.environ["OPENAI_API_KEY"]) -vector_store_driver = PineconeVectorStoreDriver(embedding_driver=embedding_driver) +vector_store_driver = PineconeVectorStoreDriver( + api_key=os.environ["PINECONE_API_KEY"], + environment=os.environ["PINECONE_ENVIRONMENT"], + index_name=os.environ['PINECONE_INDEX_NAME'], + embedding_driver=embedding_driver, +) # Load Artifacts from the web artifacts = WebLoader(max_tokens=100).load("https://www.griptape.ai") From 1cc802658721fbeee90ba5aeb5c7d217be25e406 Mon Sep 17 00:00:00 2001 From: cjkindel Date: Fri, 12 Jul 2024 14:17:19 -0700 Subject: [PATCH 04/10] make lint --- .../griptape_cloud_knowledge_base_vector_store_driver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/griptape/drivers/vector/griptape_cloud_knowledge_base_vector_store_driver.py b/griptape/drivers/vector/griptape_cloud_knowledge_base_vector_store_driver.py index 0c7d5c961..416d5e258 100644 --- a/griptape/drivers/vector/griptape_cloud_knowledge_base_vector_store_driver.py +++ b/griptape/drivers/vector/griptape_cloud_knowledge_base_vector_store_driver.py @@ -1,8 +1,10 @@ +from typing import Any, Optional from urllib.parse import urljoin + import requests -from typing import Optional, Any from attrs import Factory, define, field -from griptape.artifacts import TextArtifact, ListArtifact + +from griptape.artifacts import ListArtifact, TextArtifact from griptape.drivers import BaseEmbeddingDriver, BaseVectorStoreDriver, DummyEmbeddingDriver From e1eb135318ab73128c680e5a427fa2e3cea00bb4 Mon Sep 17 00:00:00 2001 From: cjkindel Date: Fri, 12 Jul 2024 14:21:42 -0700 Subject: [PATCH 05/10] reformat --- .../drivers/vector-store-drivers.md | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/docs/griptape-framework/drivers/vector-store-drivers.md b/docs/griptape-framework/drivers/vector-store-drivers.md index c86bb6a5c..1d594f777 100644 --- a/docs/griptape-framework/drivers/vector-store-drivers.md +++ b/docs/griptape-framework/drivers/vector-store-drivers.md @@ -40,11 +40,7 @@ artifacts = WebLoader(max_tokens=100).load("https://www.griptape.ai") # Upsert Artifacts into the Vector Store Driver [vector_store_driver.upsert_text_artifact(a, namespace="griptape") for a in artifacts] -results = vector_store_driver.query( - "creativity", - count=3, - namespace="griptape" -) +results = vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] @@ -67,7 +63,7 @@ gt_cloud_knowledge_base_id = os.environ["GRIPTAPE_CLOUD_KB_ID"] vector_store_driver = GriptapeCloudKnowledgeBaseVectorStoreDriver(api_key=gt_cloud_api_key, knowledge_base_id=gt_cloud_knowledge_base_id) -results =vector_store_driver.query(query="What is griptape?") +results = vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] @@ -106,11 +102,7 @@ artifacts = WebLoader(max_tokens=100).load("https://www.griptape.ai") # Upsert Artifacts into the Vector Store Driver [vector_store_driver.upsert_text_artifact(a, namespace="griptape") for a in artifacts] -results = vector_store_driver.query( - "creativity", - count=3, - namespace="griptape" -) +results = vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] @@ -156,7 +148,7 @@ vector_store_driver.upsert_text_artifacts( } ) -results =vector_store_driver.query(query="What is griptape?") +results = vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] @@ -208,7 +200,7 @@ vector_store_driver.upsert_text_artifacts( } ) -results =vector_store_driver.query(query="What is griptape?") +results = vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] @@ -279,7 +271,7 @@ vector_store_driver.upsert_text_artifacts( } ) -results =vector_store_driver.query(query="What is griptape?") +results = vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] @@ -322,7 +314,7 @@ vector_store_driver.upsert_text_artifacts( } ) -results =vector_store_driver.query(query="What is griptape?") +results = vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] @@ -369,7 +361,7 @@ vector_store_driver.upsert_text_artifacts( } ) -results =vector_store_driver.query(query="What is griptape?") +results = vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] @@ -431,7 +423,7 @@ vector_store_driver.upsert_text_artifacts( } ) -results =vector_store_driver.query(query="What is griptape?") +results = vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] @@ -496,7 +488,7 @@ vector_store_driver.upsert_vector( content=artifacts[0].value ) -results =vector_store_driver.query(query="What is griptape?") +results = vector_store_driver.query(query="What is griptape?") values = [r.to_artifact().value for r in results] From bd5142ba2eaf2b45ba3d6c42b88cab943f01465c Mon Sep 17 00:00:00 2001 From: cjkindel Date: Fri, 12 Jul 2024 14:27:13 -0700 Subject: [PATCH 06/10] handle none count for qdrant --- griptape/drivers/vector/qdrant_vector_store_driver.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/griptape/drivers/vector/qdrant_vector_store_driver.py b/griptape/drivers/vector/qdrant_vector_store_driver.py index d3038839e..4c47e7916 100644 --- a/griptape/drivers/vector/qdrant_vector_store_driver.py +++ b/griptape/drivers/vector/qdrant_vector_store_driver.py @@ -87,7 +87,7 @@ def delete_vector(self, vector_id: str) -> None: def query( self, query: str, - count: Optional[int] = BaseVectorStoreDriver.DEFAULT_QUERY_COUNT, + count: Optional[int] = None, namespace: Optional[str] = None, include_vectors: bool = False, **kwargs, @@ -106,7 +106,13 @@ def query( query_vector = self.embedding_driver.embed_string(query) # Create a search request - results = self.client.search(collection_name=self.collection_name, query_vector=query_vector, limit=count) + request = { + "collection_name": self.collection_name, + "query_vector": query_vector, + "limit": count, + } + request = {k: v for k, v in request.items() if v is not None} + results = self.client.search(**request) # Convert results to QueryResult objects query_results = [ From c313f0870ef3e69e4b02d7c544c83c916c788f46 Mon Sep 17 00:00:00 2001 From: cjkindel Date: Fri, 12 Jul 2024 14:29:50 -0700 Subject: [PATCH 07/10] make format --- griptape/drivers/vector/qdrant_vector_store_driver.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/griptape/drivers/vector/qdrant_vector_store_driver.py b/griptape/drivers/vector/qdrant_vector_store_driver.py index 4c47e7916..a5162f754 100644 --- a/griptape/drivers/vector/qdrant_vector_store_driver.py +++ b/griptape/drivers/vector/qdrant_vector_store_driver.py @@ -106,11 +106,7 @@ def query( query_vector = self.embedding_driver.embed_string(query) # Create a search request - request = { - "collection_name": self.collection_name, - "query_vector": query_vector, - "limit": count, - } + request = {"collection_name": self.collection_name, "query_vector": query_vector, "limit": count} request = {k: v for k, v in request.items() if v is not None} results = self.client.search(**request) From b185a7b00bfbe8eb52b54fe29e870daaa335169d Mon Sep 17 00:00:00 2001 From: cjkindel Date: Fri, 12 Jul 2024 15:48:05 -0700 Subject: [PATCH 08/10] Use standard example for qdrant snippet --- .../drivers/vector-store-drivers.md | 32 +++++-------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/docs/griptape-framework/drivers/vector-store-drivers.md b/docs/griptape-framework/drivers/vector-store-drivers.md index 1d594f777..e0d3226e1 100644 --- a/docs/griptape-framework/drivers/vector-store-drivers.md +++ b/docs/griptape-framework/drivers/vector-store-drivers.md @@ -24,7 +24,6 @@ The [LocalVectorStoreDriver](../../reference/griptape/drivers/vector/local_vecto ```python import os -from griptape.artifacts import BaseArtifact from griptape.drivers import LocalVectorStoreDriver, OpenAiEmbeddingDriver from griptape.loaders import WebLoader @@ -53,7 +52,6 @@ The [GriptapeCloudKnowledgeBaseVectorStoreDriver](../../reference/griptape/drive ```python import os -from griptape.artifacts import BaseArtifact from griptape.drivers import GriptapeCloudKnowledgeBaseVectorStoreDriver @@ -81,7 +79,6 @@ Here is an example of how the Driver can be used to load and query information i ```python import os -from griptape.artifacts import BaseArtifact from griptape.drivers import PineconeVectorStoreDriver, OpenAiEmbeddingDriver from griptape.loaders import WebLoader @@ -92,7 +89,7 @@ embedding_driver = OpenAiEmbeddingDriver(api_key=os.environ["OPENAI_API_KEY"]) vector_store_driver = PineconeVectorStoreDriver( api_key=os.environ["PINECONE_API_KEY"], environment=os.environ["PINECONE_ENVIRONMENT"], - index_name=os.environ['PINECONE_INDEX_NAME'], + index_name=os.environ["PINECONE_INDEX_NAME"], embedding_driver=embedding_driver, ) @@ -446,47 +443,34 @@ from griptape.tokenizers import HuggingFaceTokenizer from griptape.loaders import WebLoader # Set up environment variables -embedding_model_name = "sentence-transformers/all-MiniLM-L6-v2" host = os.environ["QDRANT_CLUSTER_ENDPOINT"] -huggingface_token = os.environ["HUGGINGFACE_HUB_ACCESS_TOKEN"] +api_key = os.environ["QDRANT_CLUSTER_API_KEY"] -# Initialize HuggingFace Embedding Driver -embedding_driver = HuggingFaceHubEmbeddingDriver( - api_token=huggingface_token, - model=embedding_model_name, - tokenizer=HuggingFaceTokenizer(model=embedding_model_name, max_output_tokens=512), -) +# Initialize an Embedding Driver. +embedding_driver = OpenAiEmbeddingDriver(api_key=os.environ["OPENAI_API_KEY"]) -# Initialize Qdrant Vector Store Driver vector_store_driver = QdrantVectorStoreDriver( url=host, collection_name="griptape", content_payload_key="content", embedding_driver=embedding_driver, - api_key=os.environ["QDRANT_CLUSTER_API_KEY"], + api_key=api_key, ) # Load Artifacts from the web artifacts = WebLoader().load("https://www.griptape.ai") -# Encode text to get embeddings -embeddings = embedding_driver.embed_text_artifact(artifacts[0]) - # Recreate Qdrant collection vector_store_driver.client.recreate_collection( collection_name=vector_store_driver.collection_name, vectors_config={ - "size": len(embeddings), + "size": 1536, "distance": vector_store_driver.distance }, ) -# Upsert vector into Qdrant -vector_store_driver.upsert_vector( - vector=embeddings, - vector_id=str(artifacts[0].id), - content=artifacts[0].value -) +# Upsert Artifacts into the Vector Store Driver +[vector_store_driver.upsert_text_artifact(a, namespace="griptape") for a in artifacts] results = vector_store_driver.query(query="What is griptape?") From dbcd45fdfe55b0f8ac3efd71abb50a96667e70c5 Mon Sep 17 00:00:00 2001 From: cjkindel Date: Fri, 12 Jul 2024 15:51:29 -0700 Subject: [PATCH 09/10] document fix --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd83a793c..402ce99c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed ### Fixed +- Parameter `count` for `QdrantVectorStoreDriver.query` now optional as per documentation. ## [0.28.1] - 2024-07-10 From c66e05aaf1dbe6b79ad483a6a8dbc59b1af091ed Mon Sep 17 00:00:00 2001 From: cjkindel Date: Fri, 12 Jul 2024 16:46:22 -0700 Subject: [PATCH 10/10] fix qd imports --- docs/griptape-framework/drivers/vector-store-drivers.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/griptape-framework/drivers/vector-store-drivers.md b/docs/griptape-framework/drivers/vector-store-drivers.md index e0d3226e1..c1f54ab2e 100644 --- a/docs/griptape-framework/drivers/vector-store-drivers.md +++ b/docs/griptape-framework/drivers/vector-store-drivers.md @@ -438,8 +438,7 @@ Here is an example of how the Driver can be used to query information in a Qdran ```python import os -from griptape.drivers import QdrantVectorStoreDriver, HuggingFaceHubEmbeddingDriver -from griptape.tokenizers import HuggingFaceTokenizer +from griptape.drivers import QdrantVectorStoreDriver, OpenAiEmbeddingDriver from griptape.loaders import WebLoader # Set up environment variables