feat(monorepo): Merge rust services into Posthog as a subtree

.github/workflows/build-and-deploy-prod.yml

@@ -7,6 +7,8 @@ on:
     push:
         branches:
             - master
+        paths-ignore:
+            - 'rust/**'
 
 jobs:
     slack:

.github/workflows/ci-hog.yml

@@ -51,7 +51,7 @@ jobs:
             # If this run wasn't initiated by the bot (meaning: snapshot update) and we've determined
             # there are backend changes, cancel previous runs
             - uses: n1hility/cancel-previous-runs@v3
-              if: github.actor != 'posthog-bot' && needs.changes.outputs.backend == 'true'
+              if: github.actor != 'posthog-bot' && needs.changes.outputs.hog == 'true'
               with:
                   token: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/container-images-cd.yml

@@ -13,6 +13,8 @@ on:
     push:
         branches:
             - master
+        paths-ignore:
+            - 'rust/**'
     workflow_dispatch:
 
 jobs:

.github/workflows/rust-docker-build.yml

@@ -0,0 +1,92 @@
+name: Build container images
+
+on:
+    workflow_dispatch:
+    push:
+        paths:
+            - 'rust/**'
+        branches:
+            - 'master'
+
+jobs:
+    build:
+        name: Build and publish container image
+        strategy:
+            matrix:
+                image:
+                    - capture
+                    - hook-api
+                    - hook-janitor
+                    - hook-worker
+        runs-on: depot-ubuntu-22.04-4
+        permissions:
+            id-token: write # allow issuing OIDC tokens for this workflow run
+            contents: read # allow reading the repo contents
+            packages: write # allow push to ghcr.io
+
+        defaults:
+            run:
+                working-directory: rust
+
+        steps:
+            - name: Check Out Repo
+              # Checkout project code
+              # Use sparse checkout to only select files in rust directory
+              # Turning off cone mode ensures that files in the project root are not included during checkout
+              uses: actions/checkout@v3
+              with:
+                  sparse-checkout: 'rust/'
+                  sparse-checkout-cone-mode: false
+
+            - name: Set up Depot CLI
+              uses: depot/setup-action@v1
+
+            - name: Login to DockerHub
+              uses: docker/login-action@v2
+              with:
+                  username: posthog
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+            - name: Login to ghcr.io
+              uses: docker/login-action@v2
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+                  logout: false
+
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v2
+
+            - name: Docker meta
+              id: meta
+              uses: docker/metadata-action@v4
+              with:
+                  images: ghcr.io/posthog/posthog/${{ matrix.image }}
+                  tags: |
+                      type=ref,event=pr
+                      type=ref,event=branch
+                      type=semver,pattern={{version}}
+                      type=semver,pattern={{major}}.{{minor}}
+                      type=sha
+
+            - name: Set up Docker Buildx
+              id: buildx
+              uses: docker/setup-buildx-action@v2
+
+            - name: Build and push image
+              id: docker_build
+              uses: depot/build-push-action@v1
+              with:
+                  context: ./rust/
+                  file: ./rust/Dockerfile
+                  push: true
+                  tags: ${{ steps.meta.outputs.tags }}
+                  labels: ${{ steps.meta.outputs.labels }}
+                  platforms: linux/arm64
+                  cache-from: type=gha
+                  cache-to: type=gha,mode=max
+                  build-args: BIN=${{ matrix.image }}
+
+            - name: Container image digest
+              run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/rust-hook-migrator-docker.yml

@@ -0,0 +1,86 @@
+name: Build hook-migrator docker image
+
+on:
+    workflow_dispatch:
+    push:
+        paths:
+            - 'rust/**'
+        branches:
+            - 'master'
+
+permissions:
+    packages: write
+
+jobs:
+    build:
+        name: build and publish hook-migrator image
+        runs-on: depot-ubuntu-22.04-4
+        permissions:
+            id-token: write # allow issuing OIDC tokens for this workflow run
+            contents: read # allow reading the repo contents
+            packages: write # allow push to ghcr.io
+
+        defaults:
+            run:
+                working-directory: rust
+
+        steps:
+            - name: Check Out Repo
+              # Checkout project code
+              # Use sparse checkout to only select files in rust directory
+              # Turning off cone mode ensures that files in the project root are not included during checkout
+              uses: actions/checkout@v3
+              with:
+                  sparse-checkout: 'rust/'
+                  sparse-checkout-cone-mode: false
+
+            - name: Set up Depot CLI
+              uses: depot/setup-action@v1
+
+            - name: Login to DockerHub
+              uses: docker/login-action@v2
+              with:
+                  username: posthog
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+            - name: Login to ghcr.io
+              uses: docker/login-action@v2
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v2
+
+            - name: Docker meta
+              id: meta
+              uses: docker/metadata-action@v4
+              with:
+                  images: ghcr.io/posthog/hog-rs/hook-migrator
+                  tags: |
+                      type=ref,event=pr
+                      type=ref,event=branch
+                      type=semver,pattern={{version}}
+                      type=semver,pattern={{major}}.{{minor}}
+                      type=sha
+
+            - name: Set up Docker Buildx
+              id: buildx
+              uses: docker/setup-buildx-action@v2
+
+            - name: Build and push migrator
+              id: docker_build_hook_migrator
+              uses: depot/build-push-action@v1
+              with:
+                  context: ./rust/
+                  file: ./rust/Dockerfile.migrate
+                  push: true
+                  tags: ${{ steps.meta.outputs.tags }}
+                  labels: ${{ steps.meta.outputs.labels }}
+                  platforms: linux/arm64
+                  cache-from: type=gha
+                  cache-to: type=gha,mode=max
+
+            - name: Hook-migrator image digest
+              run: echo ${{ steps.docker_build_hook_migrator.outputs.digest }}

.github/workflows/rust.yml

@@ -0,0 +1,192 @@
+name: Rust
+
+on:
+    workflow_dispatch:
+    push:
+        branches: [master, main]
+    pull_request:
+
+env:
+    CARGO_TERM_COLOR: always
+
+jobs:
+    # Job to decide if we should run rust ci
+    # See https://github.com/dorny/paths-filter#conditional-execution for more details
+    changes:
+        runs-on: ubuntu-latest
+        timeout-minutes: 5
+        if: github.repository == 'PostHog/posthog'
+        name: Determine need to run rust checks
+        # Set job outputs to values from filter step
+        outputs:
+            rust: ${{ steps.filter.outputs.rust }}
+        steps:
+            # For pull requests it's not necessary to checkout the code, but we
+            # also want this to run on master so we need to checkout
+            - uses: actions/checkout@v3
+            - uses: dorny/paths-filter@v2
+              id: filter
+              with:
+                  filters: |
+                      rust:
+                        # Avoid running rust tests for irrelevant changes
+                        - 'rust/**'
+
+    build:
+        needs: changes
+        runs-on: depot-ubuntu-22.04-4
+
+        defaults:
+            run:
+                working-directory: rust
+
+        steps:
+            # Checkout project code
+            # Use sparse checkout to only select files in rust directory
+            # Turning off cone mode ensures that files in the project root are not included during checkout
+            - uses: actions/checkout@v3
+              if: needs.changes.outputs.rust == 'true'
+              with:
+                  sparse-checkout: 'rust/'
+                  sparse-checkout-cone-mode: false
+
+            - name: Install rust
+              if: needs.changes.outputs.rust == 'true'
+              uses: dtolnay/[email protected]
+
+            - uses: actions/cache@v3
+              if: needs.changes.outputs.rust == 'true'
+              with:
+                  path: |
+                      ~/.cargo/registry
+                      ~/.cargo/git
+                      rust/target
+                  key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }}
+
+            - name: Run cargo build
+              if: needs.changes.outputs.rust == 'true'
+              run: cargo build --all --locked --release && find target/release/ -maxdepth 1 -executable -type f | xargs strip
+
+    test:
+        needs: changes
+        runs-on: depot-ubuntu-22.04-4
+        timeout-minutes: 10
+
+        defaults:
+            run:
+                working-directory: rust
+
+        steps:
+            # Checkout project code
+            # Use sparse checkout to only select files in rust directory
+            # Turning off cone mode ensures that files in the project root are not included during checkout
+            - uses: actions/checkout@v3
+              if: needs.changes.outputs.rust == 'true'
+              with:
+                  sparse-checkout: 'rust/'
+                  sparse-checkout-cone-mode: false
+
+            - name: Login to DockerHub
+              if: needs.changes.outputs.rust == 'true'
+              uses: docker/login-action@v2
+              with:
+                  username: posthog
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+            - name: Setup dependencies
+              if: needs.changes.outputs.rust == 'true'
+              run: |
+                  docker compose up kafka redis db echo_server -d --wait
+                  docker compose up setup_test_db
+                  echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
+
+            - name: Install rust
+              if: needs.changes.outputs.rust == 'true'
+              uses: dtolnay/[email protected]
+
+            - uses: actions/cache@v3
+              if: needs.changes.outputs.rust == 'true'
+              with:
+                  path: |
+                      ~/.cargo/registry
+                      ~/.cargo/git
+                      rust/target
+                  key: ${ runner.os }-cargo-debug-${{ hashFiles('**/Cargo.lock') }}
+
+            - name: Run cargo test
+              if: needs.changes.outputs.rust == 'true'
+              run: cargo test --all-features
+
+    linting:
+        needs: changes
+        runs-on: depot-ubuntu-22.04-4
+
+        defaults:
+            run:
+                working-directory: rust
+
+        steps:
+            # Checkout project code
+            # Use sparse checkout to only select files in rust directory
+            # Turning off cone mode ensures that files in the project root are not included during checkout
+            - uses: actions/checkout@v3
+              if: needs.changes.outputs.rust == 'true'
+              with:
+                  sparse-checkout: 'rust/'
+                  sparse-checkout-cone-mode: false
+
+            - name: Install rust
+              if: needs.changes.outputs.rust == 'true'
+              uses: dtolnay/[email protected]
+              with:
+                  components: clippy,rustfmt
+
+            - uses: actions/cache@v3
+              if: needs.changes.outputs.rust == 'true'
+              with:
+                  path: |
+                      ~/.cargo/registry
+                      ~/.cargo/git
+                      rust/target
+                  key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}
+
+            - name: Check format
+              if: needs.changes.outputs.rust == 'true'
+              run: cargo fmt -- --check
+
+            - name: Run clippy
+              if: needs.changes.outputs.rust == 'true'
+              run: cargo clippy -- -D warnings
+
+            - name: Run cargo check
+              if: needs.changes.outputs.rust == 'true'
+              run: cargo check --all-features
+
+    shear:
+        needs: changes
+        runs-on: depot-ubuntu-22.04-4
+
+        defaults:
+            run:
+                working-directory: rust
+
+        steps:
+            # Checkout project code
+            # Use sparse checkout to only select files in rust directory
+            # Turning off cone mode ensures that files in the project root are not included during checkout
+            - uses: actions/checkout@v3
+              if: needs.changes.outputs.rust == 'true'
+              with:
+                  sparse-checkout: 'rust/'
+                  sparse-checkout-cone-mode: false
+
+            - name: Install cargo-binstall
+              if: needs.changes.outputs.rust == 'true'
+              uses: cargo-bins/cargo-binstall@main
+
+            - name: Install cargo-shear
+              if: needs.changes.outputs.rust == 'true'
+              run: cargo binstall --no-confirm cargo-shear
+
+            - run: cargo shear
+              if: needs.changes.outputs.rust == 'true'