From 81375e82992b67d02d579d8bda1637975f0f0503 Mon Sep 17 00:00:00 2001 From: Hammad Bashir Date: Thu, 21 Mar 2024 15:33:00 -0700 Subject: [PATCH] [BUG] Rename memberlist to match new memberlist name (#1910) ## Description of changes *Summarize the changes made by this PR.* - Improvements & Bug fixes - We renamed the memberlist to query-service-memberlist given the query/compactor split. - This aligns all the defaults with that name - New functionality - None ## Test plan *How are these changes tested?* - [x] Tests pass locally with `pytest` for python, `yarn test` for js, `cargo test` for rust ## Documentation Changes None --- chromadb/config.py | 6 +++--- chromadb/segment/impl/distributed/server.py | 9 ++------- go/cmd/coordinator/cmd.go | 7 ++++--- rust/worker/chroma_config.yaml | 2 +- rust/worker/src/memberlist/memberlist_provider.rs | 2 +- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/chromadb/config.py b/chromadb/config.py index 9f9c2f50e45..d0e6e45a00f 100644 --- a/chromadb/config.py +++ b/chromadb/config.py @@ -112,7 +112,7 @@ class Settings(BaseSettings): # type: ignore chroma_collection_assignment_policy_impl: str = ( "chromadb.ingest.impl.simple_policy.SimpleAssignmentPolicy" ) - worker_memberlist_name: str = "worker-memberlist" + worker_memberlist_name: str = "query-service-memberlist" chroma_coordinator_host = "localhost" chroma_logservice_host = "localhost" @@ -331,11 +331,11 @@ def __init__(self, settings: Settings): and settings["chroma_segment_cache_policy"] != "LRU" ): logger.error( - f"Failed to set chroma_segment_cache_policy: Only LRU is available." + "Failed to set chroma_segment_cache_policy: Only LRU is available." ) if settings["chroma_memory_limit_bytes"] == 0: logger.error( - f"Failed to set chroma_segment_cache_policy: chroma_memory_limit_bytes is require." + "Failed to set chroma_segment_cache_policy: chroma_memory_limit_bytes is require." ) # Apply the nofile limit if set diff --git a/chromadb/segment/impl/distributed/server.py b/chromadb/segment/impl/distributed/server.py index d9a6c317f7a..32bd1f67cfd 100644 --- a/chromadb/segment/impl/distributed/server.py +++ b/chromadb/segment/impl/distributed/server.py @@ -11,13 +11,8 @@ import chromadb.proto.chroma_pb2 as proto import grpc from concurrent import futures -from chromadb.proto.convert import ( - to_proto_vector_embedding_record -) from chromadb.segment import SegmentImplementation, SegmentType -from chromadb.telemetry.opentelemetry import ( - OpenTelemetryClient -) +from chromadb.telemetry.opentelemetry import OpenTelemetryClient from chromadb.types import EmbeddingRecord from chromadb.segment.distributed import MemberlistProvider, Memberlist from chromadb.utils.rendezvous_hash import assign, murmur3hasher @@ -54,7 +49,7 @@ def __init__(self, system: System) -> None: self._opentelemetry_client = system.require(OpenTelemetryClient) # TODO: add term and epoch to segment server self._memberlist_provider = system.require(MemberlistProvider) - self._memberlist_provider.set_memberlist_name("worker-memberlist") + self._memberlist_provider.set_memberlist_name("query-service-memberlist") self._assignment_policy = system.require(CollectionAssignmentPolicy) self._create_pulsar_topics() self._consumer = system.require(Consumer) diff --git a/go/cmd/coordinator/cmd.go b/go/cmd/coordinator/cmd.go index d549adb7eae..918515fdfa6 100644 --- a/go/cmd/coordinator/cmd.go +++ b/go/cmd/coordinator/cmd.go @@ -1,11 +1,12 @@ package main import ( - "github.com/chroma-core/chroma/go/pkg/coordinator/grpc" - "github.com/chroma-core/chroma/go/pkg/grpcutils" "io" "time" + "github.com/chroma-core/chroma/go/pkg/coordinator/grpc" + "github.com/chroma-core/chroma/go/pkg/grpcutils" + "github.com/chroma-core/chroma/go/cmd/flag" "github.com/chroma-core/chroma/go/pkg/utils" "github.com/spf13/cobra" @@ -53,7 +54,7 @@ func init() { // Memberlist Cmd.Flags().StringVar(&conf.KubernetesNamespace, "kubernetes-namespace", "chroma", "Kubernetes namespace") - Cmd.Flags().StringVar(&conf.WorkerMemberlistName, "worker-memberlist-name", "worker-memberlist", "Worker memberlist name") + Cmd.Flags().StringVar(&conf.WorkerMemberlistName, "worker-memberlist-name", "query-service-memberlist", "Worker memberlist name") Cmd.Flags().StringVar(&conf.AssignmentPolicy, "assignment-policy", "rendezvous", "Assignment policy") Cmd.Flags().DurationVar(&conf.WatchInterval, "watch-interval", 60*time.Second, "Watch interval") } diff --git a/rust/worker/chroma_config.yaml b/rust/worker/chroma_config.yaml index 151d2128779..b0f8c80aceb 100644 --- a/rust/worker/chroma_config.yaml +++ b/rust/worker/chroma_config.yaml @@ -16,7 +16,7 @@ worker: hasher: Murmur3 memberlist_provider: CustomResource: - memberlist_name: "worker-memberlist" + memberlist_name: "query-service-memberlist" queue_size: 100 ingest: queue_size: 10000 diff --git a/rust/worker/src/memberlist/memberlist_provider.rs b/rust/worker/src/memberlist/memberlist_provider.rs index 4233f514931..c6adfbd3f38 100644 --- a/rust/worker/src/memberlist/memberlist_provider.rs +++ b/rust/worker/src/memberlist/memberlist_provider.rs @@ -258,7 +258,7 @@ mod tests { let kube_ns = "chroma".to_string(); let kube_client = Client::try_default().await.unwrap(); let memberlist_provider = CustomResourceMemberlistProvider::new( - "worker-memberlist".to_string(), + "query-service-memberlist".to_string(), kube_client.clone(), kube_ns.clone(), 10,