Merge branch 'java_benchmarks' into java_benchmarks_tlsflag

Signed-off-by: acarbonetto <[email protected]>

.github/workflows/install-redis/action.yml

@@ -16,7 +16,7 @@ runs:
         - run: mkdir ~/redis-binaries
           shell: bash
 
-        - uses: actions/checkout@v3
+        - uses: actions/checkout@v4
           with:
               submodules: recursive
 

.github/workflows/java-benchmark.yml

@@ -0,0 +1,46 @@
+name: Java client benchmarks
+
+on:
+  workflow_dispatch:
+    inputs:
+      name:
+        required: false
+        type: string
+
+run-name: ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }}
+
+jobs:
+  java-benchmark:
+    strategy:
+      matrix:
+        java:
+          - 11
+          - 17
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: "temurin"
+          java-version: ${{ matrix.java }}
+
+      - name: Start Redis
+        run: docker run -p 6379:6379 -p 8001:8001 -d redis/redis-stack
+
+      - name: Run benchmarks
+        working-directory: java/jabushka
+        run: ./gradlew :benchmark:run
+
+      - name: Upload test reports
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-reports-${{ matrix.java }}
+          path: |
+            java/jabushka/benchmarks/build/reports/**

.github/workflows/java.yml

@@ -0,0 +1,63 @@
+name: SQL Java CI
+
+on:
+  pull_request:
+  push:
+    paths:
+      - "java/**"
+      - ".github/workflows/java.yml"
+
+# Run only most latest job on a branch and cancel previous ones
+concurrency:
+  group: ${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-and-test-java-client:
+    strategy:
+      # Run all jobs
+      fail-fast: false
+      matrix:
+        java:
+          - 11
+          - 17
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: "temurin"
+          java-version: ${{ matrix.java }}
+
+      - name: Install and run protoc (protobuf)
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+          mkdir -p java/jabushka/jabushka/src/main/java/org/babushka/jabushka/generated
+          protoc -Iprotobuf=babushka-core/src/protobuf/ --java_out=java/jabushka/jabushka/src/main/java/org/babushka/jabushka/generated babushka-core/src/protobuf/*.proto
+
+      - name: Build rust part
+        working-directory: java
+        run: cargo build
+
+      - name: Start Redis
+        run: docker run -p 6379:6379 -p 8001:8001 -d redis/redis-stack
+
+      - name: Build java part
+        working-directory: java/jabushka
+        run: ./gradlew --continue build
+
+      - name: Upload test reports
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-reports-${{ matrix.java }}
+          path: |
+            java/jabushka/jabushka/build/reports/**
+            java/jabushka/integTest/build/reports/**

.github/workflows/lint-rust/action.yml

@@ -10,38 +10,31 @@ runs:
     using: "composite"
 
     steps:
-        - uses: actions/checkout@v3
+        - uses: actions/checkout@v4
           with:
               submodules: recursive
-
-        - uses: actions-rs/toolchain@v1
-          with:
-              profile: minimal
-              toolchain: stable
-              override: true
-              components: rustfmt, clippy
 
-        - uses: Swatinem/rust-cache@v1
-
+        - uses: dtolnay/rust-toolchain@stable
+        - uses: Swatinem/rust-cache@v2
+
         - run: |
-            sudo apt update
-            sudo apt install protobuf-compiler
+              sudo apt update
+              sudo apt install protobuf-compiler
+          shell: bash
+
+        - run: cargo fmt --all -- --check
+          working-directory: ${{ inputs.cargo-toml-folder }}
+          shell: bash
+
+        - run: cargo clippy --all-features --all-targets -- -D warnings
+          working-directory: ${{ inputs.cargo-toml-folder }}
           shell: bash
-        - uses: actions-rs/cargo@v1
-          with:
-              command: fmt
-              args: --manifest-path ${{ inputs.cargo-toml-folder }}/Cargo.toml --all -- --check
 
-        - uses: actions-rs/cargo@v1
-          with:
-              command: clippy
-              args: --manifest-path ${{ inputs.cargo-toml-folder }}/Cargo.toml --all-features --all-targets -- -D warnings
-
         - run: |
-              cd ${{ inputs.cargo-toml-folder }}
               cargo update
               cargo install cargo-deny
               cargo deny check licenses --config ${GITHUB_WORKSPACE}/deny.toml
+          working-directory: ${{ inputs.cargo-toml-folder }}
           shell: bash
 
         - name: doc

.github/workflows/lint-ts.yml

@@ -5,10 +5,12 @@ on:
         paths:
             - benchmarks/node/**
             - node/**
+            - benchmarks/utilities/*
     pull_request:
         paths:
             - benchmarks/node/**
             - node/**
+            - benchmarks/utilities/*
 
 env:
     CARGO_TERM_COLOR: always
@@ -17,7 +19,7 @@ jobs:
         runs-on: ubuntu-latest
         timeout-minutes: 10
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
             - name: lint ts
               run: |
                   npm install @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-tsdoc eslint typescript

.github/workflows/node.yml

@@ -28,7 +28,7 @@ jobs:
                     - 7.2.0
 
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
@@ -67,10 +67,11 @@ jobs:
         timeout-minutes: 15
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
             - uses: ./.github/workflows/lint-rust
               with:
                   cargo-toml-folder: ./node/rust-client
+              name: lint node rust

.github/workflows/python.yml

@@ -28,7 +28,7 @@ jobs:
                     - 7.2.0
 
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
@@ -51,7 +51,7 @@ jobs:
               working-directory: ./python
               run: |
                   python -m pip install --upgrade pip
-                  pip install flake8 isort black
+                  pip install flake8 isort black mypy-protobuf
 
             - name: Lint with isort
               working-directory: ./python
@@ -78,25 +78,43 @@ jobs:
             - name: Generate protobuf files
               working-directory: .
               run: |
-                  protoc -Iprotobuf=./babushka-core/src/protobuf/ --python_out=./python/python/pybushka ./babushka-core/src/protobuf/*.proto
+                  MYPY_PROTOC_PATH=`which protoc-gen-mypy`
+                  protoc --plugin=protoc-gen-mypy=${MYPY_PROTOC_PATH} -Iprotobuf=./babushka-core/src/protobuf/ --python_out=./python/python/pybushka --mypy_out=./python/python/pybushka ./babushka-core/src/protobuf/*.proto
 
-            - name: Test with pytest
+            - name: Build Pybushka
               working-directory: ./python
               run: |
                   python -m venv .env
                   source .env/bin/activate
                   pip install -r requirements.txt
                   maturin develop --release
+
+            - name: Type check with mypy
+              working-directory: ./python
+              run: |
+                  # The type check should run inside the virtual env to get
+                  # all installed dependencies and build files
+                  source .env/bin/activate
+                  pip install mypy types-protobuf
+                  # Install the benchmark requirements 
+                  pip install -r ../benchmarks/python/requirements.txt
+                  python -m mypy ..
+              
+            - name: Test with pytest
+              working-directory: ./python
+              run: |
+                  source .env/bin/activate
                   pytest --asyncio-mode=auto
 
     lint-rust:
         runs-on: ubuntu-latest
         timeout-minutes: 15
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
             - uses: ./.github/workflows/lint-rust
               with:
-                  cargo-toml-folder: ./python
+                  cargo-toml-folder: ./python                  
+              name: lint python-rust

.github/workflows/rust.yml

@@ -28,7 +28,7 @@ jobs:
                     - 7.2.0
 
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
@@ -42,14 +42,8 @@ jobs:
                   sudo apt update
                   sudo apt install protobuf-compiler
 
-            - name: Install toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  toolchain: stable
-                  override: true
-                  components: rustfmt
-
-            - uses: Swatinem/rust-cache@v1
+            - uses: dtolnay/rust-toolchain@stable
+            - uses: Swatinem/rust-cache@v2
 
             - name: Run tests
               working-directory: ./babushka-core
@@ -69,18 +63,21 @@ jobs:
         runs-on: ubuntu-latest
         timeout-minutes: 20
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
             - uses: ./.github/workflows/lint-rust
               with:
                   cargo-toml-folder: ./babushka-core
+              name: lint babushka-core
 
             - uses: ./.github/workflows/lint-rust
               with:
                   cargo-toml-folder: ./logger_core
+              name: lint logger
 
             - uses: ./.github/workflows/lint-rust
               with:
                   cargo-toml-folder: ./benchmarks/rust
+              name: lint benchmark

.gitignore

@@ -4,6 +4,9 @@
 debug/
 target/
 
+# Git stuff
+.worktrees
+
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
 Cargo.lock
@@ -17,6 +20,14 @@ dump.rdb
 .env
 benchmarks/results
 
+# IDE generaged files
+.vs
+.vscode
+.idea
+
+# MacOS metadata
+.DS_Store
+
 # lock files
 
 yarn.lock

README.md

@@ -23,9 +23,9 @@ Babushka (temporary name, final name pending) is a collection of open source Red
 
 -   GCC
 -   pkg-config
--   protobuf-compiler (protoc)
+-   protobuf-compiler (protoc) >= V3
 -   openssl
--   libssl-dev
+-   libssl-dev // for amazon-linux install openssl-devel
 -   python3
 
 Installation for ubuntu:

babushka-core/.cargo/config.toml

@@ -0,0 +1,3 @@
+[env]
+BABUSHKA_NAME = "Babushka" # This should be overwritten by each wrapper library.
+BABUSHKA_VERSION = "0.1.0"

babushka-core/Cargo.toml

@@ -15,12 +15,12 @@ signal-hook = "^0.3"
 signal-hook-tokio = {version = "^0.3", features = ["futures-v0_3"] }
 tokio = { version = "1", features = ["macros", "time"] }
 logger_core = {path = "../logger_core"}
-dispose = "0.4.0"
+dispose = "0.5.0"
 tokio-util = {version = "^0.7", features = ["rt"]}
 num_cpus = "^1.15"
 tokio-retry = "0.3.0"
 protobuf = {version= "3", features = ["bytes", "with-bytes"]}
-integer-encoding = "3.0.4"
+integer-encoding = "4.0.0"
 thiserror = "1"
 rand = "0.8.5"
 futures-intrusive = "0.5.0"
@@ -35,7 +35,7 @@ criterion = { version = "^0.5", features = ["html_reports", "async_tokio"] }
 which = "4.4.0"
 ctor = "0.2.2"
 redis = { path = "../submodules/redis-rs/redis", features = ["tls-rustls-insecure"] }
-iai-callgrind = "0.3.1"
+iai-callgrind = "0.6"
 tokio = { version = "1", features = ["rt-multi-thread"] }
 once_cell = "1.18.0"