feat(cdp): Use cyclotron part 2 (#24746)

.github/workflows/ci-plugin-server.yml

@@ -57,7 +57,6 @@ jobs:
         defaults:
             run:
                 working-directory: 'plugin-server'
-
         steps:
             - uses: actions/checkout@v3
 
@@ -82,6 +81,7 @@ jobs:
     tests:
         name: Plugin Server Tests (${{matrix.shard}})
         needs: changes
+        if: needs.changes.outputs.plugin-server == 'true'
         runs-on: ubuntu-latest
 
         strategy:
@@ -97,21 +97,17 @@ jobs:
 
         steps:
             - name: Code check out
-              if: needs.changes.outputs.plugin-server == 'true'
               uses: actions/checkout@v3
 
             - name: Stop/Start stack with Docker Compose
-              if: needs.changes.outputs.plugin-server == 'true'
               run: |
                   docker compose -f docker-compose.dev.yml down
                   docker compose -f docker-compose.dev.yml up -d
 
             - name: Add Kafka to /etc/hosts
-              if: needs.changes.outputs.plugin-server == 'true'
               run: echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
 
             - name: Set up Python
-              if: needs.changes.outputs.plugin-server == 'true'
               uses: actions/setup-python@v5
               with:
                   python-version: 3.11.9
@@ -122,50 +118,57 @@ jobs:
             # uv is a fast pip alternative: https://github.com/astral-sh/uv/
             - run: pip install uv
 
+            - name: Install rust
+              uses: dtolnay/[email protected]
+
+            - uses: actions/cache@v4
+              with:
+                  path: |
+                      ~/.cargo/registry
+                      ~/.cargo/git
+                      rust/target
+                  key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }}
+
+            - name: Install sqlx-cli
+              working-directory: rust
+              run: cargo install [email protected] --no-default-features --features native-tls,postgres
+
             - name: Install SAML (python3-saml) dependencies
-              if: needs.changes.outputs.plugin-server == 'true'
               run: |
                   sudo apt-get update
                   sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
 
             - name: Install python dependencies
-              if: needs.changes.outputs.plugin-server == 'true'
               run: |
                   uv pip install --system -r requirements-dev.txt
                   uv pip install --system -r requirements.txt
 
             - name: Install pnpm
-              if: needs.changes.outputs.plugin-server == 'true'
               uses: pnpm/action-setup@v4
 
             - name: Set up Node.js
-              if: needs.changes.outputs.plugin-server == 'true'
               uses: actions/setup-node@v4
               with:
                   node-version: 18.12.1
                   cache: pnpm
                   cache-dependency-path: plugin-server/pnpm-lock.yaml
 
             - name: Install package.json dependencies with pnpm
-              if: needs.changes.outputs.plugin-server == 'true'
               run: cd plugin-server && pnpm i
 
             - name: Wait for Clickhouse, Redis & Kafka
-              if: needs.changes.outputs.plugin-server == 'true'
               run: |
                   docker compose -f docker-compose.dev.yml up kafka redis clickhouse -d --wait
                   bin/check_kafka_clickhouse_up
 
             - name: Set up databases
-              if: needs.changes.outputs.plugin-server == 'true'
               env:
                   TEST: 'true'
                   SECRET_KEY: 'abcdef' # unsafe - for testing only
                   DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
               run: cd plugin-server && pnpm setup:test
 
             - name: Test with Jest
-              if: needs.changes.outputs.plugin-server == 'true'
               env:
                   # Below DB name has `test_` prepended, as that's how Django (ran above) creates the test DB
                   DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/test_posthog'

.vscode/launch.json

@@ -119,7 +119,8 @@
                 "WORKER_CONCURRENCY": "2",
                 "OBJECT_STORAGE_ENABLED": "True",
                 "HOG_HOOK_URL": "http://localhost:3300/hoghook",
-                "CDP_ASYNC_FUNCTIONS_RUSTY_HOOK_TEAMS": ""
+                "CDP_ASYNC_FUNCTIONS_RUSTY_HOOK_TEAMS": "",
+                "CDP_CYCLOTRON_ENABLED_TEAMS": "*"
             },
             "presentation": {
                 "group": "main"

bin/migrate

@@ -1,5 +1,11 @@
 #!/bin/bash
 set -e
+SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
+
+# NOTE when running in docker, rust might not exist so we need to check for it
+if [ -d "$SCRIPT_DIR/../rust" ]; then
+    bash $SCRIPT_DIR/../rust/bin/migrate-cyclotron
+fi
 
 python manage.py migrate
 python manage.py migrate_clickhouse

bin/start-cyclotron

@@ -12,7 +12,7 @@ export RUST_LOG=${DEBUG:-debug}
 SQLX_QUERY_LEVEL=${SQLX_QUERY_LEVEL:-warn}
 export RUST_LOG=$RUST_LOG,sqlx::query=$SQLX_QUERY_LEVEL
 
-export DATABASE_URL=${DATABASE_URL:-postgres://posthog:posthog@localhost:5432/posthog}
+export DATABASE_URL=${CYCLOTRON_DATABASE_URL:-postgres://posthog:posthog@localhost:5432/cyclotron}
 export ALLOW_INTERNAL_IPS=${ALLOW_INTERNAL_IPS:-true}
 
 ./target/debug/cyclotron-fetch &

plugin-server/package.json

@@ -23,7 +23,8 @@
         "prettier:check": "prettier --check .",
         "prepublishOnly": "pnpm build",
         "setup:dev:clickhouse": "cd .. && DEBUG=1 python manage.py migrate_clickhouse",
-        "setup:test": "cd .. && TEST=1 python manage.py setup_test_environment",
+        "setup:test": "cd .. && TEST=1 python manage.py setup_test_environment && cd plugin-server && pnpm run setup:test:cyclotron",
+        "setup:test:cyclotron": "CYCLOTRON_DATABASE_NAME=test_cyclotron ../rust/bin/migrate-cyclotron",
         "services:start": "cd .. && docker compose -f docker-compose.dev.yml up",
         "services:stop": "cd .. && docker compose -f docker-compose.dev.yml down",
         "services:clean": "cd .. && docker compose -f docker-compose.dev.yml rm -v",

plugin-server/src/cdp/cdp-api.ts

@@ -9,7 +9,7 @@ import { HogExecutor } from './hog-executor'
 import { HogFunctionManager } from './hog-function-manager'
 import { HogWatcher, HogWatcherState } from './hog-watcher'
 import { HogFunctionInvocationResult, HogFunctionType, LogEntry } from './types'
-import { createInvocation } from './utils'
+import { createInvocation, queueBlobToString } from './utils'
 
 export class CdpApi {
     private hogExecutor: HogExecutor
@@ -144,11 +144,19 @@ export class CdpApi {
                 if (invocation.queue === 'fetch') {
                     if (mock_async_functions) {
                         // Add the state, simulating what executeAsyncResponse would do
+
+                        // Re-parse the fetch args for the logging
+                        const fetchArgs = {
+                            ...invocation.queueParameters,
+                            body: queueBlobToString(invocation.queueBlob),
+                        }
+
                         response = {
                             invocation: {
                                 ...invocation,
                                 queue: 'hog',
-                                queueParameters: { response: { status: 200, body: {} } },
+                                queueParameters: { response: { status: 200 } },
+                                queueBlob: Buffer.from('{}'),
                             },
                             finished: false,
                             logs: [
@@ -160,7 +168,7 @@ export class CdpApi {
                                 {
                                     level: 'info',
                                     timestamp: DateTime.now(),
-                                    message: `fetch(${JSON.stringify(invocation.queueParameters, null, 2)})`,
+                                    message: `fetch(${JSON.stringify(fetchArgs, null, 2)})`,
                                 },
                             ],
                         }