[TST] Move cluster test to bigger machine, Hardcode tests to use rend…

…ezvous hash of topics (#1651)

## Description of changes

*Summarize the changes made by this PR.*
 - Improvements & Bug fixes
- Move cluster tests to a bigger machine. I suspect the default machines
are too slow resulting in a time out.
- Fix the coordinator sysdb tests to use rendezvous hashing since we
cannot parametrize the service during integration tests easily.

.github/workflows/chroma-cluster-test.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         python: ['3.8']
-        platform: [ubuntu-latest]
+        platform: ['16core-64gb-ubuntu-latest']
         testfile: ["chromadb/test/ingest/test_producer_consumer.py",
                    "chromadb/test/db/test_system.py",
                    "chromadb/test/segment/distributed/test_memberlist_provider.py",]

bin/cluster-test.sh

@@ -26,6 +26,7 @@ minikube addons enable ingress-dns -p chroma-test
 eval $(minikube -p chroma-test docker-env)
 docker build -t server:latest -f Dockerfile .
 docker build -t chroma-coordinator:latest -f go/coordinator/Dockerfile .
+docker build -t worker -f rust/worker/Dockerfile . --build-arg CHROMA_KUBERNETES_INTEGRATION=1
 
 # Apply the kubernetes manifests
 kubectl apply -f k8s/deployment
@@ -45,8 +46,8 @@ sleep 10
 
 export CHROMA_CLUSTER_TEST_ONLY=1
 export CHROMA_SERVER_HOST=$(kubectl get svc server -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
-export PULSAR_BROKER_URL=$(kubectl get svc pulsar -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
-export CHROMA_COORDINATOR_HOST=$(kubectl get svc coordinator -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
+export PULSAR_BROKER_URL=$(kubectl get svc pulsar-lb -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
+export CHROMA_COORDINATOR_HOST=$(kubectl get svc coordinator-lb -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
 export CHROMA_SERVER_GRPC_PORT="50051"
 
 echo "Chroma Server is running at port $CHROMA_SERVER_HOST"

chromadb/test/db/test_system.py

@@ -25,11 +25,17 @@
 
 # These are the sample collections that are used in the tests below. Tests can override
 # the fields as needed.
+
+# HACK: In order to get the real grpc tests passing, we need the topic to use rendezvous
+# hashing. This is because the grpc tests use the real grpc sysdb server and the
+# rendezvous hashing is done in the segment server. We don't have a easy way to parameterize
+# the assignment policy in the grpc tests, so we just use rendezvous hashing for all tests.
+# by harcoding the topic to what we expect rendezvous hashing to return with 16 topics.
 sample_collections = [
     Collection(
         id=uuid.UUID(int=1),
         name="test_collection_1",
-        topic=f"persistent://{PULSAR_TENANT}/{PULSAR_NAMESPACE}/00000000-0000-0000-0000-000000000001",
+        topic=f"persistent://{PULSAR_TENANT}/{PULSAR_NAMESPACE}/chroma_log_1",
         metadata={"test_str": "str1", "test_int": 1, "test_float": 1.3},
         dimension=128,
         database=DEFAULT_DATABASE,
@@ -38,7 +44,7 @@
     Collection(
         id=uuid.UUID(int=2),
         name="test_collection_2",
-        topic=f"persistent://{PULSAR_TENANT}/{PULSAR_NAMESPACE}/00000000-0000-0000-0000-000000000002",
+        topic=f"persistent://{PULSAR_TENANT}/{PULSAR_NAMESPACE}/chroma_log_14",
         metadata={"test_str": "str2", "test_int": 2, "test_float": 2.3},
         dimension=None,
         database=DEFAULT_DATABASE,
@@ -47,7 +53,7 @@
     Collection(
         id=uuid.UUID(int=3),
         name="test_collection_3",
-        topic=f"persistent://{PULSAR_TENANT}/{PULSAR_NAMESPACE}/00000000-0000-0000-0000-000000000003",
+        topic=f"persistent://{PULSAR_TENANT}/{PULSAR_NAMESPACE}/chroma_log_14",
         metadata={"test_str": "str3", "test_int": 3, "test_float": 3.3},
         dimension=None,
         database=DEFAULT_DATABASE,
@@ -174,7 +180,7 @@ def test_create_get_delete_collections(sysdb: SysDB) -> None:
     # Find by topic
     for collection in sample_collections:
         result = sysdb.get_collections(topic=collection["topic"])
-        assert result == [collection]
+        assert collection in result
 
     # Find by id
     for collection in sample_collections: