[ENH]: add config map based config file for rust worker (#2110)

## Description of changes
*Summarize the changes made by this PR.*
 - Improvements & Bug fixes
	 - add config map based config file for rust worker

Tiltfile

@@ -122,6 +122,7 @@ k8s_resource(
     'test-memberlist:MemberList',
     'test-memberlist-reader:ClusterRole',
     'test-memberlist-reader-binding:ClusterRoleBinding',
+    'compaction-service-config:configmap',
   ],
   new_name='k8s_setup',
   labels=["infrastructure"],

k8s/distributed-chroma/configuration/default.yaml

@@ -0,0 +1,64 @@
+# Default configuration for query and compaction service
+# In the long term, every service should have an entry in this file
+# and this can become the global configuration file for Chroma
+# for now we nest it in the worker directory
+
+query_service:
+    my_ip: "10.244.0.9"
+    my_port: 50051
+    assignment_policy:
+        RendezvousHashing:
+            hasher: Murmur3
+    memberlist_provider:
+        CustomResource:
+            kube_namespace: "chroma"
+            memberlist_name: "query-service-memberlist"
+            queue_size: 100
+    sysdb:
+        Grpc:
+            host: "sysdb.chroma"
+            port: 50051
+    storage:
+        S3:
+            bucket: "chroma-storage"
+            credentials: "Minio"
+    log:
+        Grpc:
+            host: "logservice.chroma"
+            port: 50051
+    dispatcher:
+        num_worker_threads: 4
+        dispatcher_queue_size: 100
+        worker_queue_size: 100
+
+compaction_service:
+    my_ip: "10.244.0.9"
+    my_port: 50051
+    assignment_policy:
+        RendezvousHashing:
+            hasher: Murmur3
+    memberlist_provider:
+        CustomResource:
+            kube_namespace: "chroma"
+            memberlist_name: "compaction-service-memberlist"
+            queue_size: 100
+    sysdb:
+        Grpc:
+            host: "sysdb.chroma"
+            port: 50051
+    storage:
+        S3:
+            bucket: "chroma-storage"
+            credentials: "Minio"
+    log:
+        Grpc:
+            host: "logservice.chroma"
+            port: 50051
+    dispatcher:
+        num_worker_threads: 4
+        dispatcher_queue_size: 100
+        worker_queue_size: 100
+    compactor:
+        compaction_manager_queue_size: 1000
+        max_concurrent_jobs: 100
+        compaction_interval_sec: 60

k8s/distributed-chroma/templates/compaction-service.yaml

@@ -1,3 +1,17 @@
+{{if .Values.compactionService.configFilePath}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: compaction-service-config
+  namespace: {{ .Values.namespace }}
+data:
+  config.yaml: |
+{{ (.Files.Get .Values.compactionService.configFilePath) | indent 4 }}
+---
+{{ end }}
+
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -15,13 +29,28 @@ spec:
         member-type: compaction-service
     spec:
       serviceAccountName: compaction-service-serviceaccount
+      {{if .Values.compactionService.configFilePath}}
+      volumes:
+        - name: compaction-service-config
+          configMap:
+            name: compaction-service-config
+      {{ end }}
       containers:
         - name: compaction-service
           image: "{{ .Values.compactionService.image.repository }}:{{ .Values.compactionService.image.tag }}"
           imagePullPolicy: IfNotPresent
+          {{if .Values.compactionService.configFilePath}}
+          volumeMounts:
+            - name: compaction-service-config
+              mountPath: /config/
+          {{ end }}
           ports:
             - containerPort: 50051
           env:
+            {{if .Values.compactionService.configFilePath}}
+            - name: CONFIG_PATH
+              value: /config/config.yaml
+            {{ end }}
             {{ range .Values.compactionService.env }}
             - name: {{ .name }}
               # TODO properly use flow control here to check which type of value we need.
@@ -38,9 +67,6 @@ spec:
           labelSelector:
             matchLabels:
               member-type: compaction-service
-      volumes:
-        - name: chroma
-          emptyDir: {}
 
 ---
 

k8s/distributed-chroma/values.yaml

@@ -58,6 +58,7 @@ queryService:
   env:
 
 compactionService:
+  configFilePath: "configuration/default.yaml"
   image:
     repository: 'local'
     tag: 'compaction-service'