feat: Create a new blob-store service for handling large temporal pay…

…loads (#642)

- **feat(blob-store): Add blob store for storing large blobs for agent
workflows**
- **feat(blob-store): Add authentication**

<!-- ELLIPSIS_HIDDEN -->

----

> [!IMPORTANT]
> Introduces a new blob-store service using SeaweedFS for handling large
temporal payloads, with Docker and authentication setup.
> 
>   - **Blob Store Service**:
> - Introduces a new blob-store service using SeaweedFS in
`blob-store/Dockerfile`, `docker-compose.yml`, and
`docker-compose-ha.yml`.
> - Adds `entrypoint.sh` for environment variable checks and
configuration file generation.
>     - Includes `s3.json.template` for S3 configuration.
>   - **Environment Configuration**:
> - Updates `.env.example` with `S3_ENDPOINT`, `S3_ACCESS_KEY`, and
`S3_SECRET_KEY` for blob store configuration.
> - Adds `USE_BLOB_STORE_FOR_TEMPORAL` and `BLOB_STORE_CUTOFF_KB` for
temporal payload handling.
>   - **Authentication**:
> - Implements authentication in `s3.json.template` with access and
secret keys for the `julep` account.
>   - **Miscellaneous**:
> - Adds `.gitignore` entry for `/s3.json` in `blob-store/.gitignore`.
> - Updates main `docker-compose.yml` to include
`blob-store/docker-compose.yml`.
> 
> <sup>This description was created by </sup>[<img alt="Ellipsis"
src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=julep-ai%2Fjulep&utm_source=github&utm_medium=referral)<sup>
for 736b9df. It will automatically
update as commits are pushed.</sup>

<!-- ELLIPSIS_HIDDEN -->

---------

Signed-off-by: Diwank Singh Tomer <[email protected]>

.env.example

@@ -1,5 +1,14 @@
 # Security
 # --------
+
+# Please set these access keys to something random and unique.
+# Note: For just testing, you can set them to the same value.
+
+# On Linux, you can generate a random key with:
+# openssl rand -base64 32
+# OR
+# tr -dc 'A-Za-z0-9+_/' </dev/urandom | head -c 32; echo
+
 JWT_SHARED_KEY=<your_jwt_shared_key>
 AGENTS_API_KEY=<your_agents_api_key>
 COZO_AUTH_TOKEN=<your_cozo_auth_token>
@@ -37,6 +46,8 @@ LITELLM_REDIS_PASSWORD=<your_litellm_redis_password>
 # EMBEDDING_MODEL_ID=Alibaba-NLP/gte-large-en-v1.5
 # NUM_GPUS=1
 # INTEGRATION_SERVICE_URL=http://integrations:8000
+# USE_BLOB_STORE_FOR_TEMPORAL=false
+# BLOB_STORE_CUTOFF_KB=1024
 
 # Temporal
 # --------
@@ -69,3 +80,10 @@ LITELLM_REDIS_PASSWORD=<your_litellm_redis_password>
 # GITHUB_API_KEY=<your_github_api_key>
 # VOYAGE_API_KEY=<your_voyage_api_key>
 # GOOGLE_APPLICATION_CREDENTIALS=.keys/julep-vertexai-svc.json
+
+# Blob Store
+# -----------
+
+# S3_ENDPOINT=http://seaweedfs:8333
+# S3_ACCESS_KEY=<your_s3_access_key>
+# S3_SECRET_KEY=<your_s3_secret_key>

blob-store/.gitignore

@@ -0,0 +1 @@
+/s3.json

blob-store/Dockerfile

@@ -0,0 +1,23 @@
+# syntax=docker/dockerfile:1
+# check=error=true
+
+FROM chrislusf/seaweedfs
+
+# Install envsubst
+ENV BUILD_DEPS="gettext"  \
+    RUNTIME_DEPS="libintl"
+
+RUN set -x && \
+    apk add --update $RUNTIME_DEPS && \
+    apk add --virtual build_deps $BUILD_DEPS &&  \
+    cp /usr/bin/envsubst /usr/local/bin/envsubst && \
+    apk del build_deps
+
+# Expected environment variables:
+# - S3_ACCESS_KEY
+# - S3_SECRET_KEY
+
+COPY s3.json.template /s3.json.template
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

blob-store/docker-compose-ha.yml

@@ -0,0 +1,82 @@
+name: julep-blob-store
+
+x-seaweedfs-base:
+  &seaweedfs-base
+  image: chrislusf/seaweedfs
+  profiles:
+    - blob-store
+
+services:
+  seaweedfs-master:
+    <<: *seaweedfs-base
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=seaweedfs-master -ip.bind=0.0.0.0 -port=9333 -metricsPort=9321 -raftBootstrap"
+    healthcheck:
+      test: [ "CMD", "wget", "-qSO", "-", "http://0.0.0.0:9333/cluster/healthz" ]
+      interval: 60s
+      retries: 6
+      timeout: 60s
+      start_period: 30s
+      start_interval: 10s
+
+  seaweedfs-volume:
+    <<: *seaweedfs-base
+    ports:
+      - 28080:28080  # Since 8080 is already used by agents-api, we use 28080
+      - 18081:18080
+    command: 'volume -mserver="seaweedfs-master:9333" -dir=/data -ip.bind=0.0.0.0 -port=28080 -ip=seaweedfs-volume -metricsPort=9321 -preStopSeconds=3'
+    healthcheck:
+      test: [ "CMD", "wget", "-qSO", "-", "http://0.0.0.0:28080/healthz" ]
+      interval: 60s
+      retries: 6
+      timeout: 30s
+      start_period: 30s
+      start_interval: 10s
+
+    depends_on:
+      seaweedfs-master:
+        condition: service_healthy
+
+    volumes:
+      - seaweedfs_data:/data
+
+  seaweedfs-filer:
+    <<: *seaweedfs-base
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: 'filer -master="seaweedfs-master:9333" -ip.bind=0.0.0.0 -port=8888 -ip=seaweedfs-filer -metricsPort=9321'
+    tty: true
+    stdin_open: true
+    healthcheck:
+      test: [ "CMD", "wget", "-qSO", "-", "http://0.0.0.0:8888/healthz" ]
+      interval: 60s
+      retries: 6
+      timeout: 30s
+      start_period: 30s
+      start_interval: 10s
+
+    depends_on:
+      seaweedfs-master:
+        condition: service_healthy
+      seaweedfs-volume:
+        condition: service_healthy
+
+  seaweedfs-s3:
+    <<: *seaweedfs-base
+    ports:
+      - 8333:8333
+    command: 's3 -filer="seaweedfs-filer:8888" -ip.bind=0.0.0.0 -port=8333 -metricsPort=9321'
+    depends_on:
+      seaweedfs-master:
+        condition: service_healthy
+      seaweedfs-volume:
+        condition: service_healthy
+      seaweedfs-filer:
+        condition: service_healthy
+
+volumes:
+  seaweedfs_data:
+    external: true

blob-store/docker-compose.yml

@@ -0,0 +1,39 @@
+name: julep-blob-store
+
+services:
+  seaweedfs:
+    image: julepai/blob-store:${TAG}
+    build:
+      context: .
+      dockerfile: Dockerfile
+    profiles:
+      - blob-store
+
+    environment:
+      - S3_ACCESS_KEY=${S3_ACCESS_KEY}
+      - S3_SECRET_KEY=${S3_SECRET_KEY}
+      - DEBUG=${DEBUG:-true}
+
+    ports:
+      - 9333:9333  # master port
+      - 8333:8333  # s3 port
+      - 8888:8888  # filer port
+      - 28080:28080  # volume port
+      # - 19333:19333  # master grpc port
+      # - 18081:18080  # volume grpc port
+      # - 18888:18888  # filer grpc port
+    command: "-filer -s3 -dir=/data -ip=seaweedfs -ip.bind=0.0.0.0 -metricsPort=9321 -master.raftBootstrap=false -master.port=9333 -master.resumeState=true -volume.port=28080 -volume.index=leveldb -filer.port=8888 -s3.port=8333"
+    healthcheck:
+      test: [ "CMD", "wget", "-qSO", "-", "http://0.0.0.0:9333/cluster/healthz" ]
+      interval: 60s
+      retries: 6
+      timeout: 60s
+      start_period: 30s
+      start_interval: 10s
+
+    volumes:
+      - seaweedfs_data:/data
+
+volumes:
+  seaweedfs_data:
+    external: true

blob-store/entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -e
+
+# Check the environment variables
+for var_name in S3_ACCESS_KEY S3_SECRET_KEY
+do
+    if [ -z "`eval echo \\\$$var_name`" ]; then
+        echo "Error: Environment variable '$var_name' is not set."
+        exit 1
+    fi
+done
+
+# Generate the s3.json configuration file
+envsubst < /s3.json.template > /s3.json
+
+if [ "$DEBUG" = "true" ]; then
+    echo '@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@'
+    echo '@@@ Careful: Debug mode is enabled. @@@'
+    echo '@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@'
+
+    echo 'Printing s3.json:'
+    cat /s3.json
+fi
+
+# Forward all arguments to the seaweedfs binary
+exec weed server -s3.config=/s3.json "$@"

blob-store/s3.json.template

@@ -0,0 +1,33 @@
+{
+  "identities": [
+    {
+      "name": "anonymous",
+      "actions": [
+        "Read"
+      ]
+    },
+    {
+      "name": "julep",
+      "credentials": [
+        {
+          "accessKey": "${S3_ACCESS_KEY}",
+          "secretKey": "${S3_SECRET_KEY}"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    }
+  ],
+  "accounts": [
+      {
+        "id" : "julep",
+        "displayName": "Julep",
+        "emailAddress": "[email protected]"
+      }
+  ]
+}

docker-compose.yml

@@ -14,6 +14,7 @@ include:
   - ./integrations-service/docker-compose.yml
   - ./prometheus/docker-compose.yml
   - ./grafana/docker-compose.yml
+  - ./blob-store/docker-compose.yml
 
   # TODO: Enable after testing
   # - ./monitoring/docker-compose.yml