Merge branch 'VEC-218-update-1.0' into VEC-214-update-docker-compose-…

…files-to-reference-0-5-image
aerospike · Jul 24, 2024 · de7a449 · de7a449
2 parents be72d6f + 6a7ffa7
commit de7a449
Show file tree

Hide file tree

Showing 19 changed files with 111 additions and 153 deletions.
diff --git a/...ic-search/Dockerfile-quote-search-preview → .internal/Dockerfile-quote-search-preview b/...ic-search/Dockerfile-quote-search-preview → .internal/Dockerfile-quote-search-preview
@@ -6,8 +6,8 @@ RUN apt-get update \
     && pip3 install setuptools sql
 
 RUN mkdir /quote-search
-COPY ./quote-search /quote-search
-COPY --parents ./container-volumes/quote-search/data/quotes.csv.tgz .
+COPY ./quote-semantic-search/quote-search /quote-search
+COPY --parents ./quote-semantic-search/container-volumes/quote-search/data/quotes.csv.tgz .
 
 WORKDIR /quote-search
 RUN python3 -m pip install -r requirements.txt --extra-index-url https://aerospike.jfrog.io/artifactory/api/pypi/aerospike-pypi-dev/simple

diff --git a/prism-image-search/docker-compose-dev.yml → .internal/docker-compose-prism-dev.yml b/prism-image-search/docker-compose-dev.yml → .internal/docker-compose-prism-dev.yml
@@ -6,7 +6,7 @@ services:
     networks:
       - avs-demo
     volumes:
-      - ./container-volumes/aerospike/etc/aerospike:/opt/aerospike/etc/aerospike
+      - ./prism-image-search/container-volumes/aerospike/etc/aerospike:/opt/aerospike/etc/aerospike
     command:
       - "--config-file"
       - "/opt/aerospike/etc/aerospike/aerospike.conf"
@@ -17,7 +17,7 @@ services:
     networks:
       - avs-demo
     volumes:
-      - ./container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-vector-search
+      - ./prism-image-search/container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-proximus
   app:
     build:
       context: .
@@ -37,7 +37,7 @@ services:
       APP_NUM_QUOTES: "5000"
       GRPC_DNS_RESOLVER: native
     volumes:
-      - ./container-volumes/prism/images:/prism/static/images/data
+      - ./prism-image-search/container-volumes/prism/images:/prism/static/images/data
 
 networks:
   avs-demo: {}
diff --git a/quote-semantic-search/docker-compose-dev.yml → ...ernal/docker-compose-quote-search-dev.yml b/quote-semantic-search/docker-compose-dev.yml → ...ernal/docker-compose-quote-search-dev.yml
@@ -6,7 +6,7 @@ services:
     networks:
       - avs-demo
     volumes:
-      - ./container-volumes/aerospike/etc/aerospike:/opt/aerospike/etc/aerospike
+      - ./quote-semantic-search/container-volumes/aerospike/etc/aerospike:/opt/aerospike/etc/aerospike
     command:
       - "--config-file"
       - "/opt/aerospike/etc/aerospike/aerospike.conf"
@@ -17,7 +17,7 @@ services:
     networks:
       - avs-demo
     volumes:
-      - ./container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-vector-search
+      - ./quote-semantic-search/container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-proximus
   app:
     build:
       context: .
@@ -32,8 +32,7 @@ services:
     networks:
       - avs-demo
     volumes:
-      - ./container-volumes/quote-search/data:/container-volumes/quote-search/data
-      # ./usr/local/lib/python3.10/dist-packages/sentence_transformers
+      - ./quote-semantic-search/container-volumes/quote-search/data:/container-volumes/quote-search/data
     environment:
       AVS_HOST: avs
       AVS_PORT: "5000"

diff --git a/basic-search/requirements.txt b/basic-search/requirements.txt
@@ -1 +1 @@
-aerospike-vector-search==0.6.1
+aerospike-vector-search==1.0.0
diff --git a/prism-image-search/README.md b/prism-image-search/README.md
@@ -74,6 +74,7 @@ Add your features.conf file to container-volumes/avs/etc/avs.
 ```
 docker compose up
 ```
+
 ## Developing
 This demo is built using [Python Flask](https://flask.palletsprojects.com/en/2.3.x/)
 and [Vue.js](https://vuejs.org/). To start developing, follow the steps to 

diff --git a/...mage-search/container-volumes/avs/etc/aerospike-vector-search/aerospike-vector-search.yml b/...mage-search/container-volumes/avs/etc/aerospike-vector-search/aerospike-vector-search.yml
@@ -27,43 +27,13 @@ interconnect:
 #      port: 6001
 
 # Target Aerospike cluster
-aerospike:
+storage:
   seeds:
     - aerospike:
         port: 3000
 
-# File based credentials store only if security should be enabled.
-#security:
-#  credentials-store:
-#    type: file
-#    credentials-file: samples/credentials.yml
-#  auth-token:
-#    private-key: samples/auth/private_key.pem
-#    public-key: samples/auth/public_key.pem
-
-# Vault based credentials store only if security should be enabled.
-#security:
-#  credentials-store:
-#    type: vault
-#    url: https://vault:8200
-#    secrets-path: /secret/aerospike/aerodb1
-#    tls:
-#      key-store:
-#        store-type: PEM
-#        store-file: key.pem
-#        store-password-file: keypass.txt # Password protecting key.pem.
-#        certificate-chain-files: certchain.pem
-#      trust-store:
-#        store-type: PEM
-#        certificate-files: cacert.pem
-#  auth-token:
-#    private-key: samples/auth/private_key.pem
-#    public-key: samples/auth/public_key.pem
-
 # The logging properties.
 logging:
-  #format: json
-  #file: /var/log/aerospike-vector-search/aerospike-vector-search.log
   enable-console-logging: true
   levels:
     metrics-ticker: off
diff --git a/prism-image-search/docker-compose.yml b/prism-image-search/docker-compose.yml
@@ -12,14 +12,12 @@ services:
       - "/opt/aerospike/etc/aerospike/aerospike.conf"
   avs:
     image: aerospike/aerospike-vector-search:0.9.0
-    # ports:
-      # - "5000:5000"
     networks:
       - avs-demo
     volumes:
       - ./container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-vector-search
   app:
-    image: aerospike/prism-search-example:latest
+    image: prism:latest
     ports:
       - "8080:8080"
     networks:

diff --git a/prism-image-search/prism/indexer.py b/prism-image-search/prism/indexer.py
@@ -1,5 +1,6 @@
 import glob
 import os
+import sys
 import threading
 from multiprocessing import get_context
 from threading import Thread
@@ -23,22 +24,26 @@
 
 
 def create_index():
-    for index in avs_admin_client.index_list():
-        if (
-            index["id"]["namespace"] == Config.AVS_NAMESPACE
-            and index["id"]["name"] == Config.AVS_INDEX_NAME
-        ):
-            logger.info("Index already exists")
-            return
-
-    avs_admin_client.index_create(
-        namespace=Config.AVS_NAMESPACE,
-        name=Config.AVS_INDEX_NAME,
-        sets=Config.AVS_SET,
-        vector_field="image_embedding",
-        dimensions=MODEL_DIM,
-        vector_distance_metric=types.VectorDistanceMetric.COSINE,
-    )
+    try:
+        for index in avs_admin_client.index_list():
+            if (
+                index["id"]["namespace"] == Config.AVS_NAMESPACE
+                and index["id"]["name"] == Config.AVS_INDEX_NAME
+            ):
+                logger.info("Index already exists")
+                return
+
+        avs_admin_client.index_create(
+            namespace=Config.AVS_NAMESPACE,
+            name=Config.AVS_INDEX_NAME,
+            sets=Config.AVS_SET,
+            vector_field="image_embedding",
+            dimensions=MODEL_DIM,
+            vector_distance_metric=types.VectorDistanceMetric.COSINE,
+        )
+    except Exception as e:
+        logger.critical("Failed to connect to avs client %s", str(e))
+        sys.exit(1)
 
 
 def either(c):

diff --git a/prism-image-search/prism/requirements.txt b/prism-image-search/prism/requirements.txt
@@ -1,6 +1,6 @@
 # TODO: Include exact versions
 # Aerospike dependencies
-aerospike-vector-search==0.6.1
+aerospike-vector-search==1.0.0
 
 # Flask framework
 flask~=2.3.2

diff --git a/quote-semantic-search/.gitignore b/quote-semantic-search/.gitignore
@@ -2,4 +2,5 @@ venv/
 .venv/
 pip.conf
 container-volumes/quote-search/data/quotes.csv
-container-volumes/avs/etc/aerospike-vector-search/features.conf
+container-volumes/avs/etc/aerospike-vector-search/features.conf
+data/
diff --git a/quote-semantic-search/Dockerfile-quote-search b/quote-semantic-search/Dockerfile-quote-search
@@ -4,7 +4,6 @@ FROM ubuntu:22.04
 RUN apt-get update \
     && apt-get install -y python3 python3-pip \
     && pip3 install setuptools sql
-
 RUN mkdir /quote-search
 COPY ./quote-search /quote-search
 WORKDIR /quote-search

diff --git a/quote-semantic-search/README.md b/quote-semantic-search/README.md
@@ -70,9 +70,6 @@ This demo is build using [Python Flask](https://flask.palletsprojects.com/en/2.3
 and [Vue.js](https://vuejs.org/). In order to developer follow the steps to 
 setup your Python environment.
 
-### Setup pip
-Setup pip to use Aerospike PyPI repository following instructions [here](https://github.com/citrusleaf/aerospike-proximus-client-python/tree/main#using-the-client-from-your-application-using-pip).
-
 ### Setup Python Virtual Environment
 
 ```shell

diff --git a/...ntic-search/container-volumes/avs/etc/aerospike-vector-search/aerospike-vector-search.yml b/...ntic-search/container-volumes/avs/etc/aerospike-vector-search/aerospike-vector-search.yml
@@ -10,10 +10,11 @@ cluster:
 service:
   ports:
     5000: {}
-  # advertised-listeners:
-  #   default:
-  #     address: 127.0.0.1
-  #     port: 5002
+    # 5002:
+    #   advertised-listeners:
+    #     default:
+    #       address: 127.0.0.1
+    #       port: 5002
 
 # Management API listening ports, TLS and network interface.
 manage:
@@ -31,43 +32,13 @@ interconnect:
 #      port: 6001
 
 # Target Aerospike cluster
-aerospike:
+storage:
   seeds:
     - aerospike:
         port: 3000
 
-# File based credentials store only if security should be enabled.
-#security:
-#  credentials-store:
-#    type: file
-#    credentials-file: samples/credentials.yml
-#  auth-token:
-#    private-key: samples/auth/private_key.pem
-#    public-key: samples/auth/public_key.pem
-
-# Vault based credentials store only if security should be enabled.
-#security:
-#  credentials-store:
-#    type: vault
-#    url: https://vault:8200
-#    secrets-path: /secret/aerospike/aerodb1
-#    tls:
-#      key-store:
-#        store-type: PEM
-#        store-file: key.pem
-#        store-password-file: keypass.txt # Password protecting key.pem.
-#        certificate-chain-files: certchain.pem
-#      trust-store:
-#        store-type: PEM
-#        certificate-files: cacert.pem
-#  auth-token:
-#    private-key: samples/auth/private_key.pem
-#    public-key: samples/auth/public_key.pem
-
 # The logging properties.
 logging:
-  #format: json
-  #file: /var/log/aerospike-vector-search/aerospike-vector-search.log
   enable-console-logging: true
   levels:
-    metrics-ticker: off
+    metrics-ticker: info
diff --git a/quote-semantic-search/docker-compose.yml b/quote-semantic-search/docker-compose.yml
@@ -19,19 +19,18 @@ services:
     volumes:
       - ./container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-vector-search
   app:
-    image: aerospike/quote-search-example:latest
+    image: quote-search
     ports:
       - "8080:8080"
     networks:
       - avs-demo
     volumes:
       - ./container-volumes/quote-search/data:/container-volumes/quote-search/data
-      # ./usr/local/lib/python3.10/dist-packages/sentence_transformers
     environment:
       AVS_HOST: avs
       AVS_PORT: "5000"
       APP_NUM_QUOTES: "5000"
       GRPC_DNS_RESOLVER: native
 
 networks:
-  avs-demo: {}
+  avs-demo: {}
diff --git a/quote-semantic-search/quote-search/dataset_stats.py b/quote-semantic-search/quote-search/dataset_stats.py
@@ -4,6 +4,7 @@
 
 from avs_client import avs_client
 from config import Config
+from indexer import index_created
 
 logger = logging.getLogger(__name__)
 
@@ -18,36 +19,37 @@ def either(c):
 
 def collect_stats():
     lock.acquire()
-    try:
-        logger.info("Collecting statistics")
-        temp_counts = {}
-        not_indexed = 0
-
-        for quote_id in range(Config.NUM_QUOTES):
-            # Check if record exists
-            if avs_client.is_indexed(
-                namespace=Config.AVS_NAMESPACE,
-                set_name=Config.AVS_SET,
-                key=quote_id,
-                index_name=Config.AVS_INDEX_NAME,
-            ):
-                # Record exists
-                quote_id = str(quote_id)
-
-                if quote_id not in temp_counts:
-                    temp_counts[quote_id] = 0
-
-                temp_counts[quote_id] = temp_counts[quote_id] + 1
-            else:
-                not_indexed += 1
-
-        dataset_counts.update(temp_counts)
-
-        logger.info(
-            f"{len(dataset_counts)} quotes indexed and {not_indexed} not indexed."
-        )
-    except Exception as e:
-        logger.warn("Error collecting statistics:" + str(e))
+    if index_created:
+        try:
+            logger.info("Collecting statistics")
+            temp_counts = {}
+            not_indexed = 0
+
+            for quote_id in range(Config.NUM_QUOTES):
+                # Check if record exists
+                if avs_client.is_indexed(
+                    namespace=Config.AVS_NAMESPACE,
+                    set_name=Config.AVS_SET,
+                    key=quote_id,
+                    index_name=Config.AVS_INDEX_NAME,
+                ):
+                    # Record exists
+                    quote_id = str(quote_id)
+
+                    if quote_id not in temp_counts:
+                        temp_counts[quote_id] = 0
+
+                    temp_counts[quote_id] = temp_counts[quote_id] + 1
+                else:
+                    not_indexed += 1
+
+            dataset_counts.update(temp_counts)
+
+            logger.info(
+                f"{len(dataset_counts)} quotes indexed and {not_indexed} not indexed."
+            )
+        except Exception as e:
+            logger.warn("Error collecting statistics:" + str(e))
 
     lock.release()
     threading.Timer(30, collect_stats).start()
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		aerospike-vector-search==0.6.1
		aerospike-vector-search==1.0.0