diff --git a/Tiltfile b/Tiltfile index e91d7aa4ea3..539bb693c7f 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,25 +1,25 @@ update_settings(max_parallel_updates=6) docker_build( - 'local:migration', + 'local:sysdb-migration', context='.', dockerfile='./go/Dockerfile.migration' ) docker_build( - 'local:coordinator', + 'local:sysdb', context='.', dockerfile='./go/Dockerfile' ) docker_build( - 'local:frontend-server', + 'local:frontend-service', context='.', dockerfile='./Dockerfile', ) docker_build( - 'local:worker', + 'local:query-service', context='.', dockerfile='./rust/worker/Dockerfile' ) @@ -40,13 +40,13 @@ k8s_yaml([ # Extra stuff to make debugging and testing easier k8s_yaml([ - 'k8s/test/coordinator_service.yaml', - 'k8s/test/jaeger_service.yaml', - 'k8s/test/logservice_service.yaml', + 'k8s/test/sysdb-service.yaml', + 'k8s/test/jaeger-service.yaml', + 'k8s/test/pulsar-service.yaml', + 'k8s/test/logservice-service.yaml', 'k8s/test/minio.yaml', - 'k8s/test/pulsar_service.yaml', - 'k8s/test/worker_service.yaml', - 'k8s/test/test_memberlist_cr.yaml', + 'k8s/test/query-service-service.yaml', + 'k8s/test/test-memberlist-cr.yaml', ]) # Lots of things assume the cluster is in a basic state. Get it into a basic @@ -60,19 +60,19 @@ k8s_resource( objects=[ 'pod-watcher:Role', 'memberlists.chroma.cluster:CustomResourceDefinition', - 'worker-memberlist:MemberList', + 'query-service-memberlist:MemberList', - 'coordinator-serviceaccount:serviceaccount', - 'coordinator-serviceaccount-rolebinding:RoleBinding', - 'coordinator-worker-memberlist-binding:clusterrolebinding', + 'sysdb-serviceaccount:serviceaccount', + 'sysdb-serviceaccount-rolebinding:RoleBinding', + 'sysdb-query-service-memberlist-binding:clusterrolebinding', 'logservice-serviceaccount:serviceaccount', - 'worker-serviceaccount:serviceaccount', - 'worker-serviceaccount-rolebinding:RoleBinding', - 'worker-memberlist-readerwriter:ClusterRole', - 'worker-worker-memberlist-binding:clusterrolebinding', - 'worker-memberlist-readerwriter-binding:clusterrolebinding', + 'query-service-serviceaccount:serviceaccount', + 'query-service-serviceaccount-rolebinding:RoleBinding', + 'query-service-memberlist-readerwriter:ClusterRole', + 'query-service-query-service-memberlist-binding:clusterrolebinding', + 'query-service-memberlist-readerwriter-binding:clusterrolebinding', 'test-memberlist:MemberList', 'test-memberlist-reader:ClusterRole', @@ -84,17 +84,17 @@ k8s_resource( ) # Production Chroma -k8s_resource('postgres', resource_deps=['k8s_setup'], labels=["infrastructure"]) -k8s_resource('pulsar', resource_deps=['k8s_setup'], labels=["infrastructure"], port_forwards=['6650:6650', '8080:8080']) -k8s_resource('migration', resource_deps=['postgres'], labels=["infrastructure"]) -k8s_resource('logservice', resource_deps=['migration'], labels=["chroma"], port_forwards='50052:50051') -k8s_resource('coordinator', resource_deps=['pulsar', 'migration'], labels=["chroma"], port_forwards='50051:50051') -k8s_resource('frontend-server', resource_deps=['pulsar', 'coordinator', 'logservice'],labels=["chroma"], port_forwards='8000:8000') -k8s_resource('worker', resource_deps=['coordinator', 'pulsar'], labels=["chroma"]) +k8s_resource('postgres', resource_deps=['k8s_setup', 'namespace'], labels=["infrastructure"]) +k8s_resource('pulsar', resource_deps=['k8s_setup', 'namespace'], labels=["infrastructure"], port_forwards=['6650:6650', '8080:8080']) +k8s_resource('sysdb-migration', resource_deps=['postgres', 'namespace'], labels=["infrastructure"]) +k8s_resource('logservice', resource_deps=['sysdb-migration'], labels=["chroma"], port_forwards='50052:50051') +k8s_resource('sysdb', resource_deps=['pulsar', 'sysdb-migration'], labels=["chroma"], port_forwards='50051:50051') +k8s_resource('frontend-service', resource_deps=['pulsar', 'sysdb', 'logservice'],labels=["chroma"], port_forwards='8000:8000') +k8s_resource('query-service', resource_deps=['sysdb', 'pulsar'], labels=["chroma"]) # I have no idea why these need their own lines but the others don't. -k8s_resource(objects=['worker:service'], new_name='worker_service', resource_deps=['worker'], labels=["chroma"]) -k8s_resource(objects=['jaeger-lb:Service'], new_name='jaeger_service', resource_deps=['k8s_setup'], labels=["debug"]) +k8s_resource(objects=['query-service:service'], new_name='query-service-service', resource_deps=['query-service'], labels=["chroma"]) +k8s_resource(objects=['jaeger-lb:Service'], new_name='jaeger-service', resource_deps=['k8s_setup'], labels=["debug"]) # Local S3 k8s_resource('minio-deployment', resource_deps=['k8s_setup'], labels=["debug"], port_forwards='9000:9000') diff --git a/bin/cluster-test.sh b/bin/cluster-test.sh index 375ca464d00..5f86a78e2c6 100755 --- a/bin/cluster-test.sh +++ b/bin/cluster-test.sh @@ -11,10 +11,10 @@ echo "Chroma Server is running at port $CHROMA_SERVER_HOST" echo "Pulsar Broker is running at port $PULSAR_BROKER_URL" echo "Chroma Coordinator is running at port $CHROMA_COORDINATOR_HOST" -kubectl -n chroma port-forward svc/coordinator-lb 50051:50051 & +kubectl -n chroma port-forward svc/sysdb-lb 50051:50051 & kubectl -n chroma port-forward svc/logservice-lb 50052:50051 & kubectl -n chroma port-forward svc/pulsar-lb 6650:6650 & kubectl -n chroma port-forward svc/pulsar-lb 8080:8080 & -kubectl -n chroma port-forward svc/frontend-server 8000:8000 & +kubectl -n chroma port-forward svc/frontend-service 8000:8000 & "$@" diff --git a/k8s/distributed-chroma/templates/frontend-server.yaml b/k8s/distributed-chroma/templates/frontend-service.yaml similarity index 50% rename from k8s/distributed-chroma/templates/frontend-server.yaml rename to k8s/distributed-chroma/templates/frontend-service.yaml index 43e0b063fd0..067fd033dea 100644 --- a/k8s/distributed-chroma/templates/frontend-server.yaml +++ b/k8s/distributed-chroma/templates/frontend-service.yaml @@ -1,21 +1,21 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: frontend-server + name: frontend-service namespace: {{ .Values.namespace }} spec: replicas: 2 selector: matchLabels: - app: frontend-server + app: frontend-service template: metadata: labels: - app: frontend-server + app: frontend-service spec: containers: - - name: frontend-server - image: "{{ .Values.frontend.image.repository }}:{{ .Values.frontend.image.tag }}" + - name: frontend-service + image: "{{ .Values.frontendService.image.repository }}:{{ .Values.frontendService.image.tag }}" imagePullPolicy: IfNotPresent ports: - containerPort: 8000 @@ -24,42 +24,42 @@ spec: mountPath: /test env: - name: IS_PERSISTENT - {{ .Values.frontend.isPersistent }} + {{ .Values.frontendService.isPersistent }} - name: CHROMA_PRODUCER_IMPL - {{ .Values.frontend.producerImpl }} + {{ .Values.frontendService.producerImpl }} - name: CHROMA_CONSUMER_IMPL - {{ .Values.frontend.consumerImpl }} + {{ .Values.frontendService.consumerImpl }} - name: CHROMA_SEGMENT_MANAGER_IMPL - {{ .Values.frontend.segmentManagerImpl }} + {{ .Values.frontendService.segmentManagerImpl }} - name: PULSAR_BROKER_URL - {{ .Values.frontend.pulsarBrokerUrl }} + {{ .Values.frontendService.pulsarBrokerUrl }} - name: PULSAR_BROKER_PORT - {{ .Values.frontend.pulsarBrokerPort }} + {{ .Values.frontendService.pulsarBrokerPort }} - name: PULSAR_ADMIN_PORT - {{ .Values.frontend.pulsarAdminPort }} + {{ .Values.frontendService.pulsarAdminPort }} - name: ALLOW_RESET - {{ .Values.frontend.allowReset }} + {{ .Values.frontendService.allowReset }} - name: CHROMA_SYSDB_IMPL - {{ .Values.frontend.sysdbImpl }} + {{ .Values.frontendService.sysdbImpl }} - name: CHROMA_SERVER_GRPC_PORT - {{ .Values.frontend.serverGrpcPort }} + {{ .Values.frontendService.serverGrpcPort }} - name: CHROMA_COORDINATOR_HOST - {{ .Values.frontend.coordinatorHost }} + {{ .Values.frontendService.coordinatorHost }} - name: CHROMA_SERVER_AUTH_PROVIDER - {{ .Values.frontend.authProvider }} + {{ .Values.frontendService.authProvider }} - name: CHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER - {{ .Values.frontend.authCredentialsProvider }} + {{ .Values.frontendService.authCredentialsProvider }} - name: CHROMA_SERVER_AUTHZ_PROVIDER - {{ .Values.frontend.authzProvider }} + {{ .Values.frontendService.authzProvider }} - name: CHROMA_SERVER_AUTHZ_CONFIG_PROVIDER - {{ .Values.frontend.authzConfigProvider }} + {{ .Values.frontendService.authzConfigProvider }} - name: CHROMA_MEMBERLIST_PROVIDER_IMPL - {{ .Values.frontend.memberlistProviderImpl }} + {{ .Values.frontendService.memberlistProviderImpl }} - name: CHROMA_LOGSERVICE_HOST - {{ .Values.frontend.logServiceHost }} + {{ .Values.frontendService.logServiceHost }} - name: CHROMA_LOGSERVICE_PORT - {{ .Values.frontend.logServicePort }} -{{ .Values.frontend.otherEnvConfig | nindent 12 }} + {{ .Values.frontendService.logServicePort }} +{{ .Values.frontendService.otherEnvConfig | nindent 12 }} volumes: - name: chroma emptyDir: {} @@ -69,7 +69,7 @@ spec: apiVersion: v1 kind: Service metadata: - name: frontend-server + name: frontend-service namespace: {{ .Values.namespace }} spec: ports: @@ -77,5 +77,5 @@ spec: port: 8000 targetPort: 8000 selector: - app: frontend-server + app: frontend-service type: ClusterIP diff --git a/k8s/distributed-chroma/templates/worker.yaml b/k8s/distributed-chroma/templates/query-service.yaml similarity index 67% rename from k8s/distributed-chroma/templates/worker.yaml rename to k8s/distributed-chroma/templates/query-service.yaml index e8cee69ca0f..4ce4e06d69e 100644 --- a/k8s/distributed-chroma/templates/worker.yaml +++ b/k8s/distributed-chroma/templates/query-service.yaml @@ -3,40 +3,39 @@ apiVersion: v1 kind: Service metadata: - name: worker + name: query-service namespace: {{ .Values.namespace }} spec: ports: - - name: worker-server-port + - name: query-service-server-port port: 50051 targetPort: 50051 selector: - app: worker-server + app: query-service-server type: ClusterIP --- - apiVersion: apps/v1 kind: Deployment metadata: - name: worker + name: query-service namespace: {{ .Values.namespace }} spec: replicas: 2 selector: matchLabels: - app: worker + app: query-service template: metadata: labels: - app: worker - member-type: worker + app: query-service + member-type: query-service spec: - serviceAccountName: worker-serviceaccount + serviceAccountName: query-service-serviceaccount containers: - - name: worker - image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag }}" + - name: query-service + image: "{{ .Values.queryService.image.repository }}:{{ .Values.queryService.image.tag }}" imagePullPolicy: IfNotPresent command: ["cargo", "run"] ports: @@ -45,9 +44,9 @@ spec: - name: chroma mountPath: /index_data env: - - name: CHROMA_WORKER__PULSAR_URL + - name: CHROMA_query-service__PULSAR_URL value: pulsar://pulsar.chroma:6650 - - name: CHROMA_WORKER__MY_IP + - name: CHROMA_query-service__MY_IP valueFrom: fieldRef: fieldPath: status.podIP @@ -57,7 +56,7 @@ spec: whenUnsatisfiable: ScheduleAnyway labelSelector: matchLabels: - member-type: worker + member-type: query-service volumes: - name: chroma emptyDir: {} @@ -67,7 +66,7 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: worker-serviceaccount + name: query-service-serviceaccount namespace: {{ .Values.namespace }} --- @@ -75,7 +74,7 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: worker-serviceaccount-rolebinding + name: query-service-serviceaccount-rolebinding namespace: {{ .Values.namespace }} roleRef: apiGroup: rbac.authorization.k8s.io @@ -83,7 +82,7 @@ roleRef: name: pod-watcher subjects: - kind: ServiceAccount - name: worker-serviceaccount + name: query-service-serviceaccount namespace: {{ .Values.namespace }} --- \ No newline at end of file diff --git a/k8s/distributed-chroma/templates/migration.yaml b/k8s/distributed-chroma/templates/sysdb-migration.yaml similarity index 90% rename from k8s/distributed-chroma/templates/migration.yaml rename to k8s/distributed-chroma/templates/sysdb-migration.yaml index fcf605e2bea..daf829e74a2 100644 --- a/k8s/distributed-chroma/templates/migration.yaml +++ b/k8s/distributed-chroma/templates/sysdb-migration.yaml @@ -1,13 +1,13 @@ apiVersion: batch/v1 kind: Job metadata: - name: migration + name: sysdb-migration namespace: {{ .Values.namespace }} spec: template: metadata: labels: - app: migration + app: sysdb-migration spec: restartPolicy: OnFailure containers: @@ -19,4 +19,5 @@ spec: image: "{{ .Values.sysdbMigration.image.repository }}:{{ .Values.sysdbMigration.image.tag }}" imagePullPolicy: IfNotPresent name: migration + --- diff --git a/k8s/distributed-chroma/templates/coordinator.yaml b/k8s/distributed-chroma/templates/sysdb-service.yaml similarity index 69% rename from k8s/distributed-chroma/templates/coordinator.yaml rename to k8s/distributed-chroma/templates/sysdb-service.yaml index 37a6a2e8987..296d391c485 100644 --- a/k8s/distributed-chroma/templates/coordinator.yaml +++ b/k8s/distributed-chroma/templates/sysdb-service.yaml @@ -1,34 +1,34 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: coordinator + name: sysdb namespace: {{ .Values.namespace }} spec: - replicas: {{ .Values.coordinator.replicaCount }} + replicas: {{ .Values.sysdb.replicaCount }} selector: matchLabels: - app: coordinator + app: sysdb template: metadata: labels: - app: coordinator + app: sysdb spec: - serviceAccountName: coordinator-serviceaccount + serviceAccountName: sysdb-serviceaccount containers: - command: - "/bin/sh" - "-c" # This has to be one line to be passed into the `exec` env correctly. I truly could not tell you why. - - coordinator coordinator {{ range $k, $v := .Values.coordinator.flags }} --{{ $k }}={{ $v }} {{ end }} + - coordinator coordinator {{ range $k, $v := .Values.sysdb.flags }} --{{ $k }}={{ $v }} {{ end }} env: - {{ range .Values.coordinator.env }} + {{ range .Values.sysdb.env }} - name: {{ .name }} # TODO properly use flow control here to check which type of value we need. {{ .value | nindent 14 }} {{ end }} - image: "{{ .Values.coordinator.image.repository }}:{{ .Values.coordinator.image.tag }}" + image: "{{ .Values.sysdb.image.repository }}:{{ .Values.sysdb.image.tag }}" imagePullPolicy: IfNotPresent - name: coordinator + name: sysdb ports: - containerPort: 50051 name: grpc @@ -38,7 +38,7 @@ spec: apiVersion: v1 kind: Service metadata: - name: coordinator + name: sysdb namespace: {{ .Values.namespace }} spec: ports: @@ -46,7 +46,7 @@ spec: port: 50051 targetPort: grpc selector: - app: coordinator + app: sysdb type: ClusterIP --- @@ -54,7 +54,7 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: coordinator-serviceaccount + name: sysdb-serviceaccount namespace: {{ .Values.namespace }} --- @@ -62,7 +62,7 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: coordinator-serviceaccount-rolebinding + name: sysdb-serviceaccount-rolebinding namespace: {{ .Values.namespace }} roleRef: apiGroup: rbac.authorization.k8s.io @@ -70,7 +70,7 @@ roleRef: name: pod-watcher subjects: - kind: ServiceAccount - name: coordinator-serviceaccount + name: sysdb-serviceaccount namespace: {{ .Values.namespace }} --- diff --git a/k8s/distributed-chroma/templates/worker_memberlist_cr.yaml b/k8s/distributed-chroma/templates/worker-memberlist-cr.yaml similarity index 71% rename from k8s/distributed-chroma/templates/worker_memberlist_cr.yaml rename to k8s/distributed-chroma/templates/worker-memberlist-cr.yaml index 1b022afa2ce..2261057ec5d 100644 --- a/k8s/distributed-chroma/templates/worker_memberlist_cr.yaml +++ b/k8s/distributed-chroma/templates/worker-memberlist-cr.yaml @@ -5,7 +5,7 @@ apiVersion: chroma.cluster/v1 kind: MemberList metadata: - name: worker-memberlist + name: query-service-memberlist namespace: {{ .Values.namespace}} spec: members: @@ -15,7 +15,7 @@ spec: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: worker-memberlist-readerwriter + name: query-service-memberlist-readerwriter rules: - apiGroups: - chroma.cluster @@ -36,14 +36,14 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: coordinator-worker-memberlist-binding + name: sysdb-query-service-memberlist-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: worker-memberlist-readerwriter + name: query-service-memberlist-readerwriter subjects: - kind: ServiceAccount - name: coordinator-serviceaccount + name: sysdb-serviceaccount namespace: {{ .Values.namespace }} --- @@ -51,16 +51,16 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - # Awkward name, but this lets the worker-serviceaccount read - # the worker-memberlist. - name: worker-worker-memberlist-binding + # Awkward name, but this lets the query-service-serviceaccount read + # the query-service-memberlist. + name: query-service-query-service-memberlist-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: worker-memberlist-readerwriter + name: query-service-memberlist-readerwriter subjects: - kind: ServiceAccount - name: worker-serviceaccount + name: query-service-serviceaccount namespace: {{ .Values.namespace }} --- @@ -68,11 +68,11 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: worker-memberlist-readerwriter-binding + name: query-service-memberlist-readerwriter-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: worker-memberlist-readerwriter + name: query-service-memberlist-readerwriter subjects: - kind: ServiceAccount name: default diff --git a/k8s/distributed-chroma/values.yaml b/k8s/distributed-chroma/values.yaml index e30bcb0e58c..18476280680 100644 --- a/k8s/distributed-chroma/values.yaml +++ b/k8s/distributed-chroma/values.yaml @@ -3,10 +3,10 @@ namespace: 'chroma' -frontend: +frontendService: image: repository: 'local' - tag: 'frontend-server' + tag: 'frontend-service' # Sometimes users (and us) want to pass values directly as flags. Sometimes, these are # populated from secrets or configMaps. So we let consumers fill these directly. @@ -22,7 +22,7 @@ frontend: allowReset: 'value: "TRUE"' sysdbImpl: 'value: "chromadb.db.impl.grpc.client.GrpcSysDB"' serverGrpcPort: 'value: "50051"' - coordinatorHost: 'value: "coordinator.chroma"' + coordinatorHost: 'value: "sysdb.chroma"' authProvider: 'value: ""' authCredentialsProvider: 'value: ""' authzProvider: 'value: ""' @@ -32,10 +32,10 @@ frontend: logServicePort: 'value: "50051"' otherEnvConfig: '' -coordinator: +sysdb: image: repository: 'local' - tag: 'coordinator' + tag: 'sysdb' replicaCount: 1 env: flags: @@ -46,16 +46,16 @@ coordinator: logService: image: repository: 'local' - tag: 'coordinator' + tag: 'sysdb' env: flags: -worker: +queryService: image: repository: 'local' - tag: 'worker' + tag: 'query-service' sysdbMigration: image: repository: 'local' - tag: 'migration' + tag: 'sysdb-migration' diff --git a/k8s/test/jaeger_service.yaml b/k8s/test/jaeger-service.yaml similarity index 100% rename from k8s/test/jaeger_service.yaml rename to k8s/test/jaeger-service.yaml diff --git a/k8s/test/logservice_service.yaml b/k8s/test/logservice-service.yaml similarity index 100% rename from k8s/test/logservice_service.yaml rename to k8s/test/logservice-service.yaml diff --git a/k8s/test/pulsar_service.yaml b/k8s/test/pulsar-service.yaml similarity index 100% rename from k8s/test/pulsar_service.yaml rename to k8s/test/pulsar-service.yaml diff --git a/k8s/test/worker_service.yaml b/k8s/test/query-service-service.yaml similarity index 65% rename from k8s/test/worker_service.yaml rename to k8s/test/query-service-service.yaml index 9fac38d0e1f..6cfdd71677b 100644 --- a/k8s/test/worker_service.yaml +++ b/k8s/test/query-service-service.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: worker-lb + name: query-service-lb namespace: chroma spec: ports: - - name: worker-port + - name: query-service-port port: 50052 targetPort: 50051 selector: - app: worker + app: query-service type: LoadBalancer diff --git a/k8s/test/coordinator_service.yaml b/k8s/test/sysdb-service.yaml similarity index 79% rename from k8s/test/coordinator_service.yaml rename to k8s/test/sysdb-service.yaml index 37334b12187..dda139fcb7d 100644 --- a/k8s/test/coordinator_service.yaml +++ b/k8s/test/sysdb-service.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: coordinator-lb + name: sysdb-lb namespace: chroma spec: ports: @@ -9,5 +9,5 @@ spec: port: 50051 targetPort: 50051 selector: - app: coordinator + app: sysdb type: LoadBalancer diff --git a/k8s/test/test_memberlist_cr.yaml b/k8s/test/test-memberlist-cr.yaml similarity index 100% rename from k8s/test/test_memberlist_cr.yaml rename to k8s/test/test-memberlist-cr.yaml diff --git a/rust/worker/chroma_config.yaml b/rust/worker/chroma_config.yaml index 32e5165924d..a84e0a2d1bb 100644 --- a/rust/worker/chroma_config.yaml +++ b/rust/worker/chroma_config.yaml @@ -22,7 +22,7 @@ worker: queue_size: 10000 sysdb: Grpc: - host: "coordinator.chroma" + host: "sysdb.chroma" port: 50051 segment_manager: storage_path: "./tmp/segment_manager/"