Update to Rocksdb v9.8.4 (#164)

* update librocksdb * enable lto feature * try using latest macos * try removing sysroot * Revert "try removing sysroot" This reverts commit d271877. * Revert "try using latest macos" This reverts commit df824c3. * try remove lto flag * try to fix CI issue * try fix sysroot * try fix * use latest macos * add x86_64 target * fix macos py3.7
rocksdict · Dec 14, 2024 · 602a87d · 602a87d
1 parent 22998b7
commit 602a87d
Show file tree

Hide file tree

Showing 5 changed files with 125 additions and 19 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -5,13 +5,12 @@ on:
     types: [published]
 
 jobs:
-  macos:
+  macos-py37:
     runs-on: macos-13
     strategy:
       fail-fast: false
       matrix:
-        py_version:
-          [3.7, 3.8, 3.9, "3.10", "3.11", "3.12", "3.13", "pypy3.9", "pypy3.10"]
+        py_version: [3.7]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -40,8 +39,62 @@ jobs:
           export CC=${HOMEBREW_PREFIX}/opt/llvm@19/bin/clang
           export CXX=${HOMEBREW_PREFIX}/opt/llvm@19/bin/clang++
           export AR=${HOMEBREW_PREFIX}/opt/llvm@19/bin/llvm-ar
-          export CFLAGS="-flto=thin -O3 --sysroot=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
-          export CXXFLAGS="-flto=thin -O3 --sysroot=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
+          export CFLAGS="-flto=thin -O3 -isysroot/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
+          export CXXFLAGS="-flto=thin -O3 -isysroot/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
+          export LLD=${HOMEBREW_PREFIX}/opt/lld/bin/ld64.lld
+          export RUSTFLAGS="-Clinker-plugin-lto -Clinker=$PWD/macos-linker.sh -Clink-arg=-fuse-ld=${LLD}"
+          maturin build --release --target aarch64-apple-darwin --out dist -i ${{ matrix.py_version }}
+          maturin build --release --target x86_64-apple-darwin --out dist -i ${{ matrix.py_version }}
+      - name: Install built wheel
+        run: |
+          pip install rocksdict --no-index --find-links dist --force-reinstall || true
+          pip install speedict --no-index --find-links dist --force-reinstall || true
+      - name: Python UnitTest
+        run: |
+          python -m unittest discover -v test
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-macos-${{ matrix.py_version }}
+          path: dist
+
+  macos:
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        py_version:
+          [3.8, 3.9, "3.10", "3.11", "3.12", "3.13", "pypy3.9", "pypy3.10"]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: Install llvm-19
+        run: |
+          brew update
+          brew install llvm@19
+          brew install lld
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.py_version }}
+          architecture: arm64
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.82.0
+          default: true
+      - name: Build wheels
+        run: |
+          rustup target add x86_64-apple-darwin
+          pip install maturin
+          eval "$(brew shellenv)"
+          export MACOSX_DEPLOYMENT_TARGET=10.14
+          export LIBCLANG_PATH=${HOMEBREW_PREFIX}/opt/llvm@19/lib
+          export CC=${HOMEBREW_PREFIX}/opt/llvm@19/bin/clang
+          export CXX=${HOMEBREW_PREFIX}/opt/llvm@19/bin/clang++
+          export AR=${HOMEBREW_PREFIX}/opt/llvm@19/bin/llvm-ar
+          export CFLAGS="-flto=thin -O3 -isysroot/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
+          export CXXFLAGS="-flto=thin -O3 -isysroot/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
           export LLD=${HOMEBREW_PREFIX}/opt/lld/bin/ld64.lld
           export RUSTFLAGS="-Clinker-plugin-lto -Clinker=$PWD/macos-linker.sh -Clink-arg=-fuse-ld=${LLD}"
           maturin build --release --target aarch64-apple-darwin --out dist -i ${{ matrix.py_version }}
@@ -89,7 +142,7 @@ jobs:
         uses: messense/maturin-action@v1
         with:
           target: ${{ matrix.target }}
-          args: --release --out dist
+          args: --release --out dist --no-default-features --features bindgen-runtime
         env:
           LIBCLANG_PATH: ${{ runner.temp }}/llvm-18.1/lib
           CC: ${{ runner.temp }}/llvm-18.1/bin/clang-cl
@@ -295,7 +348,7 @@ jobs:
           rustup-components: rustfmt
           target: ${{ matrix.target }}
           manylinux: musllinux_1_2
-          args: --release --no-default-features --features bindgen-static --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10'
+          args: --release --no-default-features --features bindgen-static,lto --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10'
           container: congyuwang/musllinux_1_2_x86_64:llvm-19.1.3
           before-script-linux: |
             apk update
@@ -343,7 +396,7 @@ jobs:
           rustup-components: rustfmt
           target: ${{ matrix.target }}
           manylinux: musllinux_1_2
-          args: --release --no-default-features --features bindgen-static --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10'
+          args: --release --no-default-features --features bindgen-static,lto --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10'
           container: congyuwang/musllinux_1_2_aarch64:llvm-19.1.3
           before-script-linux: |
             apk update

diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml
@@ -5,13 +5,12 @@ on:
     branches: [main, speedict]
 
 jobs:
-  macos:
+  macos-py37:
     runs-on: macos-13
     strategy:
       fail-fast: false
       matrix:
-        py_version:
-          [3.7, 3.8, 3.9, "3.10", "3.11", "3.12", "3.13", "pypy3.9", "pypy3.10"]
+        py_version: [3.7]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -40,8 +39,62 @@ jobs:
           export CC=${HOMEBREW_PREFIX}/opt/llvm@19/bin/clang
           export CXX=${HOMEBREW_PREFIX}/opt/llvm@19/bin/clang++
           export AR=${HOMEBREW_PREFIX}/opt/llvm@19/bin/llvm-ar
-          export CFLAGS="-flto=thin -O3 --sysroot=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
-          export CXXFLAGS="-flto=thin -O3 --sysroot=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
+          export CFLAGS="-flto=thin -O3 -isysroot/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
+          export CXXFLAGS="-flto=thin -O3 -isysroot/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
+          export LLD=${HOMEBREW_PREFIX}/opt/lld/bin/ld64.lld
+          export RUSTFLAGS="-Clinker-plugin-lto -Clinker=$PWD/macos-linker.sh -Clink-arg=-fuse-ld=${LLD}"
+          maturin build --release --target aarch64-apple-darwin --out dist -i ${{ matrix.py_version }}
+          maturin build --release --target x86_64-apple-darwin --out dist -i ${{ matrix.py_version }}
+      - name: Install built wheel
+        run: |
+          pip install rocksdict --no-index --find-links dist --force-reinstall || true
+          pip install speedict --no-index --find-links dist --force-reinstall || true
+      - name: Python UnitTest
+        run: |
+          python -m unittest discover -v test
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-macos-${{ matrix.py_version }}
+          path: dist
+
+  macos:
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        py_version:
+          [3.8, 3.9, "3.10", "3.11", "3.12", "3.13", "pypy3.9", "pypy3.10"]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: Install llvm-19
+        run: |
+          brew update
+          brew install llvm@19
+          brew install lld
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.py_version }}
+          architecture: arm64
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.82.0
+          default: true
+      - name: Build wheels
+        run: |
+          rustup target add x86_64-apple-darwin
+          pip install maturin
+          eval "$(brew shellenv)"
+          export MACOSX_DEPLOYMENT_TARGET=10.14
+          export LIBCLANG_PATH=${HOMEBREW_PREFIX}/opt/llvm@19/lib
+          export CC=${HOMEBREW_PREFIX}/opt/llvm@19/bin/clang
+          export CXX=${HOMEBREW_PREFIX}/opt/llvm@19/bin/clang++
+          export AR=${HOMEBREW_PREFIX}/opt/llvm@19/bin/llvm-ar
+          export CFLAGS="-flto=thin -O3 -isysroot/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
+          export CXXFLAGS="-flto=thin -O3 -isysroot/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk"
           export LLD=${HOMEBREW_PREFIX}/opt/lld/bin/ld64.lld
           export RUSTFLAGS="-Clinker-plugin-lto -Clinker=$PWD/macos-linker.sh -Clink-arg=-fuse-ld=${LLD}"
           maturin build --release --target aarch64-apple-darwin --out dist -i ${{ matrix.py_version }}
@@ -89,7 +142,7 @@ jobs:
         uses: messense/maturin-action@v1
         with:
           target: ${{ matrix.target }}
-          args: --release --out dist
+          args: --release --out dist --no-default-features --features bindgen-runtime
         env:
           LIBCLANG_PATH: ${{ runner.temp }}/llvm-18.1/lib
           CC: ${{ runner.temp }}/llvm-18.1/bin/clang-cl
@@ -295,7 +348,7 @@ jobs:
           rustup-components: rustfmt
           target: ${{ matrix.target }}
           manylinux: musllinux_1_2
-          args: --release --no-default-features --features bindgen-static --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10'
+          args: --release --no-default-features --features bindgen-static,lto --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10'
           container: congyuwang/musllinux_1_2_x86_64:llvm-19.1.3
           before-script-linux: |
             apk update
@@ -343,7 +396,7 @@ jobs:
           rustup-components: rustfmt
           target: ${{ matrix.target }}
           manylinux: musllinux_1_2
-          args: --release --no-default-features --features bindgen-static --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10'
+          args: --release --no-default-features --features bindgen-static,lto --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10'
           container: congyuwang/musllinux_1_2_aarch64:llvm-19.1.3
           before-script-linux: |
             apk update

diff --git a/Cargo.toml b/Cargo.toml
@@ -30,9 +30,10 @@ num-bigint = "0.4"
 libc = "0.2"
 
 [features]
-default = ["bindgen-runtime"]
+default = ["bindgen-runtime", "lto"]
 bindgen-runtime = ["rocksdb/bindgen-runtime"]
 bindgen-static = ["rocksdb/bindgen-static"]
+lto = ["rocksdb/lto"]
 
 [dependencies.pyo3]
 version = "0.22"

diff --git a/rust-rocksdb b/rust-rocksdb
diff --git a/test/test_rdict.py b/test/test_rdict.py
@@ -16,7 +16,6 @@
 import sys
 import platform
 from json import loads, dumps
-from subprocess import Popen
 
 
 TEST_INT_RANGE_UPPER = 999999