From 3a90000b24f5155d167728d872bbe4ce74772b92 Mon Sep 17 00:00:00 2001 From: ijl Date: Fri, 15 Nov 2024 16:14:43 +0000 Subject: [PATCH] cargo update, build misc, macOS 15, i686 and win GHA --- .github/workflows/artifact.yaml | 152 +++++++++++--- Cargo.lock | 37 +--- Cargo.toml | 6 +- README.md | 67 +----- build.rs | 4 + ci/azure-pipelines.yml | 196 ------------------ ci/azure-win.yml | 41 ---- ci/deploy | 18 -- include/pyo3/pyo3-build-config/src/impl_.rs | 89 ++++++-- include/pyo3/pyo3-build-config/src/lib.rs | 5 + include/pyo3/pyo3-ffi/src/compat/py_3_13.rs | 21 ++ .../pyo3-ffi/src/cpython/critical_section.rs | 21 ++ .../pyo3/pyo3-ffi/src/cpython/dictobject.rs | 9 + include/pyo3/pyo3-ffi/src/datetime.rs | 65 +++--- include/pyo3/pyo3-ffi/src/lib.rs | 147 +++++-------- include/pyo3/pyo3-ffi/src/listobject.rs | 4 + include/pyo3/pyo3-ffi/src/methodobject.rs | 6 +- include/pyo3/pyo3-ffi/src/moduleobject.rs | 14 +- integration/requirements.txt | 2 +- requirements.txt | 2 +- script/install-fedora | 3 +- script/lint | 2 +- script/pymem | 34 --- src/lib.rs | 1 + src/util.rs | 6 +- test/requirements.txt | 1 - test/test_dict.py | 9 + test/test_type.py | 16 -- 28 files changed, 407 insertions(+), 571 deletions(-) delete mode 100644 ci/azure-pipelines.yml delete mode 100644 ci/azure-win.yml delete mode 100755 ci/deploy delete mode 100755 script/pymem diff --git a/.github/workflows/artifact.yaml b/.github/workflows/artifact.yaml index eae5339d..00fa1a05 100644 --- a/.github/workflows/artifact.yaml +++ b/.github/workflows/artifact.yaml @@ -11,6 +11,7 @@ jobs: sdist: runs-on: ubuntu-22.04 + timeout-minutes: 10 strategy: fail-fast: false env: @@ -77,7 +78,7 @@ jobs: { cc: "clang", cflags: "-Os -fstrict-aliasing -fno-plt -flto=full -emit-llvm", - features: "avx512,no-panic,unstable-simd,yyjson", + features: "avx512,unstable-simd,yyjson", ldflags: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow", rustflags: "-C linker=clang -C link-arg=-fuse-ld=lld -C linker-plugin-lto -C lto=fat -C link-arg=-Wl,-zrelro,-znow -Z mir-opt-level=4 -Z threads=4 -D warnings", tag: null, @@ -85,17 +86,17 @@ jobs: }, ] env: - PYTHON: "${{ matrix.python.interpreter }}" - PYTHON_PACKAGE: "${{ matrix.python.package }}" - TARGET: "${{ matrix.arch.target }}" + CARGO_TARGET_DIR: "/tmp/orjson" CC: "${{ matrix.arch.cc }}" - VENV: ".venv" - FEATURES: "${{ matrix.arch.features }}" CFLAGS: "${{ matrix.arch.cflags }}" + COMPATIBILITY: "${{ matrix.python.compatibility }}" + FEATURES: "${{ matrix.arch.features }}" LDFLAGS: "${{ matrix.arch.ldflags }}" + PYTHON: "${{ matrix.python.interpreter }}" + PYTHON_PACKAGE: "${{ matrix.python.package }}" RUSTFLAGS: "${{ matrix.arch.rustflags }}" - CARGO_TARGET_DIR: "/tmp/orjson" - COMPATIBILITY: "${{ matrix.python.compatibility }}" + TARGET: "${{ matrix.arch.target }}" + VENV: ".venv" steps: - name: cpuinfo @@ -167,7 +168,7 @@ jobs: { cc: "clang", cflags: "-Os -fstrict-aliasing -fno-plt -flto=full -emit-llvm", - features: "no-panic,unstable-simd,yyjson", + features: "unstable-simd,yyjson", ldflags: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow", rustflags: "-C linker=clang -C link-arg=-fuse-ld=lld -C linker-plugin-lto -C lto=fat -C link-arg=-Wl,-zrelro,-znow -Z mir-opt-level=4 -Z threads=4 -D warnings", tag: "aarch64", @@ -175,17 +176,17 @@ jobs: }, ] env: - PYTHON: "${{ matrix.python.interpreter }}" - PYTHON_PACKAGE: "${{ matrix.python.package }}" - TARGET: "${{ matrix.arch.target }}" + CARGO_TARGET_DIR: "/tmp/orjson" CC: "${{ matrix.arch.cc }}" - VENV: ".venv" - FEATURES: "${{ matrix.arch.features }}" CFLAGS: "${{ matrix.arch.cflags }}" + COMPATIBILITY: "${{ matrix.python.compatibility }}" + FEATURES: "${{ matrix.arch.features }}" LDFLAGS: "${{ matrix.arch.ldflags }}" + PYTHON: "${{ matrix.python.interpreter }}" + PYTHON_PACKAGE: "${{ matrix.python.package }}" RUSTFLAGS: "${{ matrix.arch.rustflags }}" - CARGO_TARGET_DIR: "/tmp/orjson" - COMPATIBILITY: "${{ matrix.python.compatibility }}" + TARGET: "${{ matrix.arch.target }}" + VENV: ".venv" steps: - name: cpuinfo @@ -241,6 +242,7 @@ jobs: musllinux: runs-on: ubuntu-24.04 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -256,11 +258,11 @@ jobs: - target: aarch64-unknown-linux-musl arch: aarch64 platform: linux/arm64 - features: no-panic,unstable-simd,unwind,yyjson + features: unstable-simd,unwind,yyjson - target: x86_64-unknown-linux-musl arch: x86_64 platform: linux/amd64 - features: avx512,no-panic,unstable-simd,unwind,yyjson + features: avx512,unstable-simd,unwind,yyjson steps: - uses: actions/checkout@v4 @@ -317,6 +319,7 @@ jobs: manylinux_non_amd64: runs-on: ubuntu-24.04 + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -331,28 +334,35 @@ jobs: { arch: 'aarch64', cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,unstable-simd,yyjson', + features: 'unstable-simd,yyjson', rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', target: 'aarch64-unknown-linux-gnu', }, + { + arch: 'i686', + cflags: '-Os -flto=full -fstrict-aliasing', + features: 'unstable-simd,yyjson', + rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', + target: 'i686-unknown-linux-gnu', + }, { arch: 'armv7', cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,yyjson', # no SIMD + features: 'yyjson', # no SIMD rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C opt-level=s', target: 'armv7-unknown-linux-gnueabihf', }, { arch: 'ppc64le', cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,unstable-simd,yyjson', + features: 'unstable-simd,yyjson', rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', target: 'powerpc64le-unknown-linux-gnu', }, { arch: 's390x', cflags: '-Os -flto=full -fstrict-aliasing -march=z10', - features: 'no-panic,yyjson', + features: 'yyjson', rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10', target: 's390x-unknown-linux-gnu', }, @@ -389,7 +399,8 @@ jobs: retention-days: 1 macos_aarch64: - runs-on: macos-14 + runs-on: macos-15 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -400,12 +411,11 @@ jobs: { version: '3.10', macosx_target: "10.15" }, ] env: - CC: "clang" CFLAGS: "-Os -fstrict-aliasing -flto=full" LDFLAGS: "-Wl,--as-needed" - CFLAGS_x86_64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -march=x86-64-v2 -mtune=generic" - CFLAGS_aarch64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -mcpu=apple-m1 -mtune=generic" - RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z threads=3 -D warnings" + CFLAGS_x86_64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -emit-llvm -march=x86-64-v2 -mtune=generic" + CFLAGS_aarch64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -emit-llvm -mcpu=apple-m1 -mtune=generic" + RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld -C linker-plugin-lto -C lto=fat -Z mir-opt-level=4 -Z threads=3 -D warnings" PATH: "/Users/runner/work/orjson/orjson/.venv/bin:/Users/runner/.cargo/bin:/usr/local/opt/curl/bin:/usr/local/bin:/usr/local/sbin:/Users/runner/bin:/Library/Frameworks/Python.framework/Versions/Current/bin:/usr/bin:/bin:/usr/sbin:/sbin" steps: @@ -414,7 +424,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "${{ matrix.python.version }}" - allow-prereleases: true - uses: dtolnay/rust-toolchain@master with: @@ -426,6 +435,10 @@ jobs: run: | cargo fetch --target aarch64-apple-darwin & + brew install llvm@19 + + export PATH=$HOME/.cargo/bin:$HOME/.local/bin:$PATH + curl -LsSf https://astral.sh/uv/install.sh | sh uv venv --python python${{ matrix.python.version }} uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt @@ -435,11 +448,15 @@ jobs: - name: maturin run: | - PATH=$HOME/.cargo/bin:$PATH \ + export PATH=$HOME/.cargo/bin:$HOME/.local/bin:$PATH + + export CC=/opt/homebrew/opt/llvm@19/bin/clang + export LLD=/opt/homebrew/opt/llvm@19/bin/lld + MACOSX_DEPLOYMENT_TARGET="${{ matrix.python.macosx_target }}" \ PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \ maturin build --release --strip \ - --features=no-panic,unstable-simd,yyjson \ + --features=unstable-simd,yyjson \ --interpreter python${{ matrix.python.version }} \ --target=universal2-apple-darwin uv pip install target/wheels/orjson*.whl @@ -463,6 +480,7 @@ jobs: macos_amd64: runs-on: macos-13 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -496,6 +514,8 @@ jobs: run: | cargo fetch --target aarch64-apple-darwin & + export PATH=$HOME/.cargo/bin:$HOME/.local/bin:$PATH + curl -LsSf https://astral.sh/uv/install.sh | sh uv venv --python python${{ matrix.python.version }} uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt @@ -505,11 +525,12 @@ jobs: - name: maturin run: | - PATH=$HOME/.cargo/bin:$PATH \ + export PATH=$HOME/.cargo/bin:$HOME/.local/bin:$PATH + MACOSX_DEPLOYMENT_TARGET="${{ matrix.python.macosx_target }}" \ PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \ maturin build --release --strip \ - --features=no-panic,unstable-simd,yyjson \ + --features=unstable-simd,yyjson \ --interpreter python${{ matrix.python.version }} \ --target=universal2-apple-darwin uv pip install target/wheels/orjson*.whl @@ -531,9 +552,75 @@ jobs: overwrite: true retention-days: 1 + windows: + runs-on: windows-2022 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.13' }, + { version: '3.12' }, + { version: '3.11' }, + { version: '3.10' }, + { version: '3.9' }, + { version: '3.8' }, + ] + platform: [ + { arch: "x64", target: "x86_64-pc-windows-msvc" }, + { arch: "x86", target: "i686-pc-windows-msvc" }, + ] + env: + CFLAGS: "-Os -flto" + LDFLAGS: "-Wl,--as-needed" + RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -D warnings" + steps: + + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "${{ matrix.python.version }}" + architecture: "${{ matrix.platform.arch }}" + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "${{ env.RUST_TOOLCHAIN }}" + targets: "${{ matrix.platform.target }}" + components: "rust-src" + + - name: Build environment + run: | + cargo fetch --target "${{ matrix.platform.target }}" & + + python.exe -m pip install --upgrade pip "maturin>=1,<2" wheel + python.exe -m pip install -r test\requirements.txt -r integration\requirements.txt + + mkdir .cargo + cp ci\config.toml .cargo\config.toml + + - name: maturin + run: | + maturin.exe build --release --strip --features=unstable-simd,yyjson --target="${{ matrix.platform.target }}" + python.exe -m pip install orjson --no-index --find-links target\wheels + + - run: python.exe -m pytest -s -rxX -v test + env: + PYTHONMALLOC: "debug" + + - name: Store wheels + if: "startsWith(github.ref, 'refs/tags/')" + uses: actions/upload-artifact@v4 + with: + name: orjson_windows_${{ matrix.platform.arch }}_${{ matrix.python.version }} + path: target\wheels + overwrite: true + retention-days: 1 + pypi: name: PyPI runs-on: ubuntu-24.04 + timeout-minutes: 10 if: "startsWith(github.ref, 'refs/tags/')" needs: [ macos_aarch64, @@ -543,6 +630,7 @@ jobs: manylinux_non_amd64, musllinux, sdist, + windows, ] environment: name: pypi diff --git a/Cargo.lock b/Cargo.lock index fde9db8d..0910646c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,9 +25,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ "shlex", ] @@ -89,9 +89,6 @@ name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" -dependencies = [ - "no-panic", -] [[package]] name = "itoap" @@ -107,9 +104,9 @@ checksum = "b9d9d414fc817d3e3d62b2598616733f76c4cc74fbac96069674739b881295c8" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.162" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" [[package]] name = "memchr" @@ -117,17 +114,6 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "no-panic" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8540b7d99a20166178b42a05776aef900cdbfec397f861dfc7819bf1d7760b3d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "once_cell" version = "1.20.2" @@ -206,24 +192,21 @@ name = "ryu" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" -dependencies = [ - "no-panic", -] [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -268,9 +251,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.86" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 516f4d0e..418df3f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,11 +38,6 @@ unstable-simd = [] # Include runtime-detected functions that use AVX512VL. Requires unstable-simd and amd64. avx512 = [] -no-panic = [ - "itoa/no-panic", - "ryu/no-panic", -] - # Avoid bundling libgcc on musl. unwind = ["unwinding"] @@ -51,6 +46,7 @@ unwind = ["unwinding"] yyjson = [] # Features detected by build.rs. Do not specify. +assert_unchecked = [] inline_int = [] intrinsics = [] optimize = [] diff --git a/README.md b/README.md index 942597ef..b8f84bbb 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,8 @@ file-like objects orjson supports CPython 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, and 3.14. -It distributes amd64/x86_64, aarch64/armv8, arm7, POWER/ppc64le, and s390x -wheels for Linux, amd64 and aarch64 wheels for macOS, and amd64 +It distributes amd64/x86_64, i686/x86, aarch64/armv8, arm7, POWER/ppc64le, +and s390x wheels for Linux, amd64 and aarch64 wheels for macOS, and amd64 and i686/x86 wheels for Windows. orjson does not and will not support PyPy, embedded Python builds for @@ -72,8 +72,7 @@ available in the repository. 3. [Testing](https://github.com/ijl/orjson?tab=readme-ov-file#testing) 4. [Performance](https://github.com/ijl/orjson?tab=readme-ov-file#performance) 1. [Latency](https://github.com/ijl/orjson?tab=readme-ov-file#latency) - 2. [Memory](https://github.com/ijl/orjson?tab=readme-ov-file#memory) - 3. [Reproducing](https://github.com/ijl/orjson?tab=readme-ov-file#reproducing) + 2. [Reproducing](https://github.com/ijl/orjson?tab=readme-ov-file#reproducing) 5. [Questions](https://github.com/ijl/orjson?tab=readme-ov-file#questions) 6. [Packaging](https://github.com/ijl/orjson?tab=readme-ov-file#packaging) 7. [License](https://github.com/ijl/orjson?tab=readme-ov-file#license) @@ -1120,64 +1119,13 @@ format, containing floats and arrays, indented. | simplejson | 16.8 | 59 | 5.6 | | json | 18.2 | 55 | 6.1 | -### Memory - -orjson as of 3.7.0 has higher baseline memory usage than other libraries -due to a persistent buffer used for parsing. Incremental memory usage when -deserializing is similar to the standard library and other third-party -libraries. - -This measures, in the first column, RSS after importing a library and reading -the fixture, and in the second column, increases in RSS after repeatedly -calling `loads()` on the fixture. - -#### twitter.json - -| Library | import, read() RSS (MiB) | loads() increase in RSS (MiB) | -|------------|----------------------------|---------------------------------| -| orjson | 15.7 | 3.4 | -| ujson | 16.4 | 3.4 | -| rapidjson | 16.6 | 4.4 | -| simplejson | 14.5 | 1.8 | -| json | 13.9 | 1.8 | - -#### github.json - -| Library | import, read() RSS (MiB) | loads() increase in RSS (MiB) | -|------------|----------------------------|---------------------------------| -| orjson | 15.2 | 0.4 | -| ujson | 15.4 | 0.4 | -| rapidjson | 15.7 | 0.5 | -| simplejson | 13.7 | 0.2 | -| json | 13.3 | 0.1 | - -#### citm_catalog.json - -| Library | import, read() RSS (MiB) | loads() increase in RSS (MiB) | -|------------|----------------------------|---------------------------------| -| orjson | 16.8 | 10.1 | -| ujson | 17.3 | 10.2 | -| rapidjson | 17.6 | 28.7 | -| simplejson | 15.8 | 30.1 | -| json | 14.8 | 20.5 | - -#### canada.json - -| Library | import, read() RSS (MiB) | loads() increase in RSS (MiB) | -|------------|----------------------------|---------------------------------| -| orjson | 17.2 | 22.1 | -| ujson | 17.4 | 18.3 | -| rapidjson | 18 | 23.5 | -| simplejson | 15.7 | 21.4 | -| json | 15.4 | 20.4 | - ### Reproducing The above was measured using Python 3.11.9 on Linux (amd64) with orjson 3.10.6, ujson 5.10.0, python-rapidson 1.18, and simplejson 3.19.2. The latency results can be reproduced using the `pybench` and `graph` -scripts. The memory results can be reproduced using the `pymem` script. +scripts. ## Questions @@ -1226,10 +1174,11 @@ example using clang and LTO. The project's own CI tests against `nightly-2024-09-25` and stable 1.72. It is prudent to pin the nightly version because that channel can introduce -breaking changes. +breaking changes. There is a significant performance benefit to using +nightly. -orjson is tested for amd64 on Linux and cross-compiles for aarch64, arm7, -ppc64le, and s390x. It is tested for either aarch64 or amd64 on macOS and +orjson is tested for amd64, aarch64, and i686 on Linux and cross-compiles for +arm7, ppc64le, and s390x. It is tested for either aarch64 or amd64 on macOS and cross-compiles for the other, depending on version. For Windows it is tested on amd64 and i686. diff --git a/build.rs b/build.rs index ad6d5b9c..089aa76c 100644 --- a/build.rs +++ b/build.rs @@ -36,6 +36,10 @@ fn main() { println!("cargo:rustc-cfg=feature=\"optimize\""); } + if let Some(true) = version_check::supports_feature("hint_assert_unchecked") { + println!("cargo:rustc-cfg=feature=\"assert_unchecked\""); + } + #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] if env::var("ORJSON_DISABLE_SIMD").is_err() { // auto build unstable SIMD on nightly diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml deleted file mode 100644 index 8382734e..00000000 --- a/ci/azure-pipelines.yml +++ /dev/null @@ -1,196 +0,0 @@ -variables: - toolchain: nightly-2024-09-25 - -jobs: - -- job: win_python313_amd64 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.13.0\x64\python.exe - rustup: https://win.rustup.rs/x86_64 - target: x86_64-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.13.0' - addToPath: true - architecture: 'x64' - - checkout: self - - template: ./azure-win.yml - -- job: win_python312_amd64 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.12.2\x64\python.exe - rustup: https://win.rustup.rs/x86_64 - target: x86_64-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12.2' - addToPath: true - architecture: 'x64' - - checkout: self - - template: ./azure-win.yml - -- job: win_python311_amd64 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.11.4\x64\python.exe - rustup: https://win.rustup.rs/x86_64 - target: x86_64-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.11.4' - addToPath: true - architecture: 'x64' - - checkout: self - - template: ./azure-win.yml - -- job: win_python310_amd64 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.10.8\x64\python.exe - rustup: https://win.rustup.rs/x86_64 - target: x86_64-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10.8' - addToPath: true - architecture: 'x64' - - checkout: self - - template: ./azure-win.yml - -- job: win_python39_amd64 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.9.13\x64\python.exe - rustup: https://win.rustup.rs/x86_64 - target: x86_64-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.9.13' - addToPath: true - architecture: 'x64' - - checkout: self - - template: ./azure-win.yml - -- job: win_python38_amd64 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.8.10\x64\python.exe - rustup: https://win.rustup.rs/x86_64 - target: x86_64-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8.10' - addToPath: true - architecture: 'x64' - - checkout: self - - template: ./azure-win.yml - -- job: win_python313_x86 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.13.0\x86\python.exe - rustup: https://win.rustup.rs/x86 - target: i686-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.13.0' - addToPath: true - architecture: 'x86' - - checkout: self - - template: ./azure-win.yml - -- job: win_python312_x86 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.12.2\x86\python.exe - rustup: https://win.rustup.rs/x86 - target: i686-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12.2' - addToPath: true - architecture: 'x86' - - checkout: self - - template: ./azure-win.yml - -- job: win_python311_x86 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.11.4\x86\python.exe - rustup: https://win.rustup.rs/x86 - target: i686-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.11.4' - addToPath: true - architecture: 'x86' - - checkout: self - - template: ./azure-win.yml - -- job: win_python310_x86 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.10.8\x86\python.exe - rustup: https://win.rustup.rs/x86 - target: i686-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10.8' - addToPath: true - architecture: 'x86' - - checkout: self - - template: ./azure-win.yml - -- job: win_python39_x86 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.9.13\x86\python.exe - rustup: https://win.rustup.rs/x86 - target: i686-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.9.13' - addToPath: true - architecture: 'x86' - - checkout: self - - template: ./azure-win.yml - -- job: win_python38_x86 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.8.10\x86\python.exe - rustup: https://win.rustup.rs/x86 - target: i686-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8.10' - addToPath: true - architecture: 'x86' - - checkout: self - - template: ./azure-win.yml diff --git a/ci/azure-win.yml b/ci/azure-win.yml deleted file mode 100644 index 655eab96..00000000 --- a/ci/azure-win.yml +++ /dev/null @@ -1,41 +0,0 @@ -parameters: - interpreter: '' - rustup: '' - target: '' - toolchain: '' - -steps: -- script: | - curl $(rustup) -o rustup-init.exe - rustup-init.exe -y --default-host $(target) --default-toolchain $(toolchain)-$(target) --profile minimal - set PATH=%PATH%;%USERPROFILE%\.cargo\bin - rustup default $(toolchain)-$(target) - rustup component add rust-src - mkdir .cargo - cp ci/config.toml .cargo/config.toml - echo "##vso[task.setvariable variable=PATH;]%PATH%;%USERPROFILE%\.cargo\bin" - displayName: rustup -- script: python.exe -m pip install --upgrade pip "maturin>=1,<2" wheel - displayName: build dependencies -- script: python.exe -m pip install -r test\requirements.txt -r integration\requirements.txt - displayName: test dependencies -- script: maturin.exe build --release --features=no-panic,unstable-simd,yyjson --strip --interpreter $(interpreter) --target $(target) - displayName: build - env: - CFLAGS: "-Os -flto" - LDFLAGS: "-Wl,--as-needed" - RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -D warnings" - CARGO_UNSTABLE_SPARSE_REGISTRY: "true" - UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" -- script: python.exe -m pip install orjson --no-index --find-links=D:\a\1\s\target\wheels - displayName: install -- script: python.exe -m pytest -s -rxX -v test - env: - PYTHONMALLOC: "debug" - displayName: pytest -- script: python.exe integration\thread - displayName: thread -- script: python.exe integration\init - displayName: init -- bash: ./ci/deploy /d/a/1/s/target/wheels/*.whl - displayName: deploy diff --git a/ci/deploy b/ci/deploy deleted file mode 100755 index 1b34b539..00000000 --- a/ci/deploy +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -if [ -z ${DRONE_TAG+x} ]; then - tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) -else - tag="$DRONE_TAG" -fi - -echo "$tag" - -if [[ "$tag" == "undefined" ]]; then - echo "not on a tag" - exit 0 -fi - -maturin upload --skip-existing "$1" diff --git a/include/pyo3/pyo3-build-config/src/impl_.rs b/include/pyo3/pyo3-build-config/src/impl_.rs index ec652591..6d232642 100644 --- a/include/pyo3/pyo3-build-config/src/impl_.rs +++ b/include/pyo3/pyo3-build-config/src/impl_.rs @@ -248,6 +248,7 @@ print("executable", sys.executable) print("calcsize_pointer", struct.calcsize("P")) print("mingw", get_platform().startswith("mingw")) print("ext_suffix", get_config_var("EXT_SUFFIX")) +print("gil_disabled", get_config_var("Py_GIL_DISABLED")) "#; let output = run_python_script(interpreter.as_ref(), SCRIPT)?; let map: HashMap = parse_script_output(&output); @@ -290,6 +291,13 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let implementation = map["implementation"].parse()?; + let gil_disabled = match map["gil_disabled"].as_str() { + "1" => true, + "0" => false, + "None" => false, + _ => panic!("Unknown Py_GIL_DISABLED value"), + }; + let lib_name = if cfg!(windows) { default_lib_name_windows( version, @@ -300,12 +308,14 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) // on Windows from sysconfig - e.g. ext_suffix may be // `_d.cp312-win_amd64.pyd` for 3.12 debug build map["ext_suffix"].starts_with("_d."), + gil_disabled, ) } else { default_lib_name_unix( version, implementation, map.get("ld_version").map(String::as_str), + gil_disabled, ) }; @@ -375,10 +385,15 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) _ => false, }; let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string); + let gil_disabled = match sysconfigdata.get_value("Py_GIL_DISABLED") { + Some(value) => value == "1", + None => false, + }; let lib_name = Some(default_lib_name_unix( version, implementation, sysconfigdata.get_value("LDVERSION"), + gil_disabled, )); let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P") .map(|bytes_width: u32| bytes_width * 8) @@ -1106,10 +1121,15 @@ impl BuildFlags { /// the interpreter and printing variables of interest from /// sysconfig.get_config_vars. fn from_interpreter(interpreter: impl AsRef) -> Result { - // sysconfig is missing all the flags on windows, so we can't actually - // query the interpreter directly for its build flags. + // sysconfig is missing all the flags on windows for Python 3.12 and + // older, so we can't actually query the interpreter directly for its + // build flags on those versions. if cfg!(windows) { - return Ok(Self::new()); + let script = String::from("import sys;print(sys.version_info < (3, 13))"); + let stdout = run_python_script(interpreter.as_ref(), &script)?; + if stdout.trim_end() == "True" { + return Ok(Self::new()); + } } let mut script = String::from("import sysconfig\n"); @@ -1528,6 +1548,7 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf abi3, false, false, + false, )) } else { None @@ -1604,9 +1625,10 @@ fn default_lib_name_for_target( abi3, false, false, + false, )) } else if is_linking_libpython_for_target(target) { - Some(default_lib_name_unix(version, implementation, None)) + Some(default_lib_name_unix(version, implementation, None, false)) } else { None } @@ -1618,16 +1640,26 @@ fn default_lib_name_windows( abi3: bool, mingw: bool, debug: bool, + gil_disabled: bool, ) -> String { if debug { // CPython bug: linking against python3_d.dll raises error // https://github.com/python/cpython/issues/101614 - format!("python{}{}_d", version.major, version.minor) + if gil_disabled { + format!("python{}{}t_d", version.major, version.minor) + } else { + format!("python{}{}_d", version.major, version.minor) + } } else if abi3 && !(implementation.is_pypy() || implementation.is_graalpy()) { WINDOWS_ABI3_LIB_NAME.to_owned() } else if mingw { + if gil_disabled { + panic!("MinGW free-threaded builds are not currently tested or supported") + } // https://packages.msys2.org/base/mingw-w64-python format!("python{}.{}", version.major, version.minor) + } else if gil_disabled { + format!("python{}{}t", version.major, version.minor) } else { format!("python{}{}", version.major, version.minor) } @@ -1637,6 +1669,7 @@ fn default_lib_name_unix( version: PythonVersion, implementation: PythonImplementation, ld_version: Option<&str>, + gil_disabled: bool, ) -> String { match implementation { PythonImplementation::CPython => match ld_version { @@ -1644,7 +1677,11 @@ fn default_lib_name_unix( None => { if version > PythonVersion::PY37 { // PEP 3149 ABI version tags are finally gone - format!("python{}.{}", version.major, version.minor) + if gil_disabled { + format!("python{}.{}t", version.major, version.minor) + } else { + format!("python{}.{}", version.major, version.minor) + } } else { // Work around https://bugs.python.org/issue36707 format!("python{}.{}m", version.major, version.minor) @@ -2351,6 +2388,7 @@ mod tests { false, false, false, + false, ), "python39", ); @@ -2361,6 +2399,7 @@ mod tests { true, false, false, + false, ), "python3", ); @@ -2371,6 +2410,7 @@ mod tests { false, true, false, + false, ), "python3.9", ); @@ -2381,6 +2421,7 @@ mod tests { true, true, false, + false, ), "python3", ); @@ -2391,6 +2432,7 @@ mod tests { true, false, false, + false, ), "python39", ); @@ -2401,6 +2443,7 @@ mod tests { false, false, true, + false, ), "python39_d", ); @@ -2413,6 +2456,7 @@ mod tests { true, false, true, + false, ), "python39_d", ); @@ -2423,16 +2467,31 @@ mod tests { use PythonImplementation::*; // Defaults to python3.7m for CPython 3.7 assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 7 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 7 }, + CPython, + None, + false + ), "python3.7m", ); // Defaults to pythonX.Y for CPython 3.8+ assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 8 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 8 }, + CPython, + None, + false + ), "python3.8", ); assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + CPython, + None, + false + ), "python3.9", ); // Can use ldversion to override for CPython @@ -2440,19 +2499,25 @@ mod tests { super::default_lib_name_unix( PythonVersion { major: 3, minor: 9 }, CPython, - Some("3.7md") + Some("3.7md"), + false ), "python3.7md", ); // PyPy 3.9 includes ldversion assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, None), + super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, None, false), "pypy3.9-c", ); assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, Some("3.9d")), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + PyPy, + Some("3.9d"), + false + ), "pypy3.9d-c", ); } diff --git a/include/pyo3/pyo3-build-config/src/lib.rs b/include/pyo3/pyo3-build-config/src/lib.rs index 6e295c99..66f52031 100644 --- a/include/pyo3/pyo3-build-config/src/lib.rs +++ b/include/pyo3/pyo3-build-config/src/lib.rs @@ -138,6 +138,10 @@ fn resolve_cross_compile_config_path() -> Option { pub fn print_feature_cfgs() { let rustc_minor_version = rustc_minor_version().unwrap_or(0); + if rustc_minor_version >= 70 { + println!("cargo:rustc-cfg=rustc_has_once_lock"); + } + // invalid_from_utf8 lint was added in Rust 1.74 if rustc_minor_version >= 74 { println!("cargo:rustc-cfg=invalid_from_utf8_lint"); @@ -175,6 +179,7 @@ pub fn print_expected_cfgs() { println!("cargo:rustc-check-cfg=cfg(pyo3_leak_on_drop_without_reference_pool)"); println!("cargo:rustc-check-cfg=cfg(diagnostic_namespace)"); println!("cargo:rustc-check-cfg=cfg(c_str_lit)"); + println!("cargo:rustc-check-cfg=cfg(rustc_has_once_lock)"); // allow `Py_3_*` cfgs from the minimum supported version up to the // maximum minor version (+1 for development for the next) diff --git a/include/pyo3/pyo3-ffi/src/compat/py_3_13.rs b/include/pyo3/pyo3-ffi/src/compat/py_3_13.rs index 9f44ced6..59289cb7 100644 --- a/include/pyo3/pyo3-ffi/src/compat/py_3_13.rs +++ b/include/pyo3/pyo3-ffi/src/compat/py_3_13.rs @@ -83,3 +83,24 @@ compat_function!( 1 } ); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyList_Extend( + list: *mut crate::PyObject, + iterable: *mut crate::PyObject, + ) -> std::os::raw::c_int { + crate::PyList_SetSlice(list, crate::PY_SSIZE_T_MAX, crate::PY_SSIZE_T_MAX, iterable) + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyList_Clear(list: *mut crate::PyObject) -> std::os::raw::c_int { + crate::PyList_SetSlice(list, 0, crate::PY_SSIZE_T_MAX, std::ptr::null_mut()) + } +); diff --git a/include/pyo3/pyo3-ffi/src/cpython/critical_section.rs b/include/pyo3/pyo3-ffi/src/cpython/critical_section.rs index 97b2f5e0..3760d224 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/critical_section.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/critical_section.rs @@ -9,6 +9,27 @@ pub struct PyCriticalSection { _cs_mutex: *mut PyMutex, } +// #[cfg(Py_GIL_DISABLED)] +// impl Default for PyCriticalSection { +// fn default() -> Self { +// let mut mutex = crate::PyMutex::new(); +// PyCriticalSection { +// _cs_prev: 0, +// _cs_mutex: core::ptr::addr_of_mut!(mutex), +// } +// } +// } + +#[cfg(Py_GIL_DISABLED)] +impl Default for PyCriticalSection { + fn default() -> Self { + PyCriticalSection { + _cs_prev: 0, + _cs_mutex: core::ptr::null_mut(), + } + } +} + #[repr(C)] #[cfg(Py_GIL_DISABLED)] pub struct PyCriticalSection2 { diff --git a/include/pyo3/pyo3-ffi/src/cpython/dictobject.rs b/include/pyo3/pyo3-ffi/src/cpython/dictobject.rs index 79dcbfdb..f67a7725 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/dictobject.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/dictobject.rs @@ -36,6 +36,15 @@ extern "C" { item: *mut PyObject, hash: crate::Py_hash_t, ) -> c_int; + + #[cfg(Py_3_13)] + pub fn _PyDict_SetItem_KnownHash_LockHeld( + mp: *mut PyDictObject, + name: *mut PyObject, + value: *mut PyObject, + hash: crate::Py_hash_t, + ) -> c_int; + // skipped _PyDict_DelItem_KnownHash // skipped _PyDict_DelItemIf // skipped _PyDict_NewKeysForClass diff --git a/include/pyo3/pyo3-ffi/src/datetime.rs b/include/pyo3/pyo3-ffi/src/datetime.rs index 7283b6d4..76d12151 100644 --- a/include/pyo3/pyo3-ffi/src/datetime.rs +++ b/include/pyo3/pyo3-ffi/src/datetime.rs @@ -4,16 +4,17 @@ //! and covers the various date and time related objects in the Python `datetime` //! standard library module. +#[cfg(not(PyPy))] +use crate::PyCapsule_Import; #[cfg(GraalPy)] use crate::{PyLong_AsLong, PyLong_Check, PyObject_GetAttrString, Py_DecRef}; use crate::{PyObject, PyObject_TypeCheck, PyTypeObject, Py_TYPE}; -use std::cell::UnsafeCell; #[cfg(not(GraalPy))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr; -#[cfg(not(PyPy))] -use {crate::PyCapsule_Import, std::ffi::CString}; +use std::sync::Once; +use std::{cell::UnsafeCell, ffi::CStr}; #[cfg(not(any(PyPy, GraalPy)))] use {crate::Py_hash_t, std::os::raw::c_uchar}; // Type struct wrappers @@ -593,6 +594,8 @@ pub struct PyDateTime_CAPI { // Python already shares this object between threads, so it's no more evil for us to do it too! unsafe impl Sync for PyDateTime_CAPI {} +pub const PyDateTime_CAPSULE_NAME: &CStr = c_str!("datetime.datetime_CAPI"); + /// Returns a pointer to a `PyDateTime_CAPI` instance /// /// # Note @@ -600,33 +603,38 @@ unsafe impl Sync for PyDateTime_CAPI {} /// `PyDateTime_IMPORT` is called #[inline] pub unsafe fn PyDateTimeAPI() -> *mut PyDateTime_CAPI { - *PyDateTimeAPI_impl.0.get() -} - -#[inline] -pub unsafe fn PyDateTime_TimeZone_UTC() -> *mut PyObject { - (*PyDateTimeAPI()).TimeZone_UTC + *PyDateTimeAPI_impl.ptr.get() } /// Populates the `PyDateTimeAPI` object pub unsafe fn PyDateTime_IMPORT() { - // PyPy expects the C-API to be initialized via PyDateTime_Import, so trying to use - // `PyCapsule_Import` will behave unexpectedly in pypy. - #[cfg(PyPy)] - let py_datetime_c_api = PyDateTime_Import(); - - #[cfg(not(PyPy))] - let py_datetime_c_api = { - // PyDateTime_CAPSULE_NAME is a macro in C - let PyDateTime_CAPSULE_NAME = CString::new("datetime.datetime_CAPI").unwrap(); - - PyCapsule_Import(PyDateTime_CAPSULE_NAME.as_ptr(), 1) as *mut PyDateTime_CAPI - }; + if !PyDateTimeAPI_impl.once.is_completed() { + // PyPy expects the C-API to be initialized via PyDateTime_Import, so trying to use + // `PyCapsule_Import` will behave unexpectedly in pypy. + #[cfg(PyPy)] + let py_datetime_c_api = PyDateTime_Import(); + + #[cfg(not(PyPy))] + let py_datetime_c_api = + PyCapsule_Import(PyDateTime_CAPSULE_NAME.as_ptr(), 1) as *mut PyDateTime_CAPI; + + if py_datetime_c_api.is_null() { + return; + } - *PyDateTimeAPI_impl.0.get() = py_datetime_c_api; + // Protect against race conditions when the datetime API is concurrently + // initialized in multiple threads. UnsafeCell.get() cannot panic so this + // won't panic either. + PyDateTimeAPI_impl.once.call_once(|| { + *PyDateTimeAPI_impl.ptr.get() = py_datetime_c_api; + }); + } } -// skipped non-limited PyDateTime_TimeZone_UTC +#[inline] +pub unsafe fn PyDateTime_TimeZone_UTC() -> *mut PyObject { + (*PyDateTimeAPI()).TimeZone_UTC +} /// Type Check macros /// @@ -739,8 +747,13 @@ extern "C" { // Rust specific implementation details -struct PyDateTimeAPISingleton(UnsafeCell<*mut PyDateTime_CAPI>); +struct PyDateTimeAPISingleton { + once: Once, + ptr: UnsafeCell<*mut PyDateTime_CAPI>, +} unsafe impl Sync for PyDateTimeAPISingleton {} -static PyDateTimeAPI_impl: PyDateTimeAPISingleton = - PyDateTimeAPISingleton(UnsafeCell::new(ptr::null_mut())); +static PyDateTimeAPI_impl: PyDateTimeAPISingleton = PyDateTimeAPISingleton { + once: Once::new(), + ptr: UnsafeCell::new(ptr::null_mut()), +}; diff --git a/include/pyo3/pyo3-ffi/src/lib.rs b/include/pyo3/pyo3-ffi/src/lib.rs index c6157401..2d0436ab 100644 --- a/include/pyo3/pyo3-ffi/src/lib.rs +++ b/include/pyo3/pyo3-ffi/src/lib.rs @@ -43,10 +43,39 @@ //! PyO3 uses `rustc`'s `--cfg` flags to enable or disable code used for different Python versions. //! If you want to do this for your own crate, you can do so with the [`pyo3-build-config`] crate. //! -//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`: Marks code that is only enabled when -//! compiling for a given minimum Python version. +//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`, `Py_3_11`, `Py_3_12`, `Py_3_13`: Marks code that is +//! only enabled when compiling for a given minimum Python version. //! - `Py_LIMITED_API`: Marks code enabled when the `abi3` feature flag is enabled. +//! - `Py_GIL_DISABLED`: Marks code that runs only in the free-threaded build of CPython. //! - `PyPy` - Marks code enabled when compiling for PyPy. +//! - `GraalPy` - Marks code enabled when compiling for GraalPy. +//! +//! Additionally, you can query for the values `Py_DEBUG`, `Py_REF_DEBUG`, +//! `Py_TRACE_REFS`, and `COUNT_ALLOCS` from `py_sys_config` to query for the +//! corresponding C build-time defines. For example, to conditionally define +//! debug code using `Py_DEBUG`, you could do: +//! +//! ```rust,ignore +//! #[cfg(py_sys_config = "Py_DEBUG")] +//! println!("only runs if python was compiled with Py_DEBUG") +//! ``` +//! +//! To use these attributes, add [`pyo3-build-config`] as a build dependency in +//! your `Cargo.toml`: +//! +//! ```toml +//! [build-dependencies] +#![doc = concat!("pyo3-build-config =\"", env!("CARGO_PKG_VERSION"), "\"")] +//! ``` +//! +//! And then either create a new `build.rs` file in the project root or modify +//! the existing `build.rs` file to call `use_pyo3_cfgs()`: +//! +//! ```rust,ignore +//! fn main() { +//! pyo3_build_config::use_pyo3_cfgs(); +//! } +//! ``` //! //! # Minimum supported Rust and Python versions //! @@ -79,101 +108,23 @@ //! [dependencies.pyo3-ffi] #![doc = concat!("version = \"", env!("CARGO_PKG_VERSION"), "\"")] //! features = ["extension-module"] +//! +//! [build-dependencies] +//! # This is only necessary if you need to configure your build based on +//! # the Python version or the compile-time configuration for the interpreter. +#![doc = concat!("pyo3_build_config = \"", env!("CARGO_PKG_VERSION"), "\"")] //! ``` //! -//! **`src/lib.rs`** -//! ```rust -//! use std::os::raw::c_char; -//! use std::ptr; -//! -//! use pyo3_ffi::*; -//! -//! static mut MODULE_DEF: PyModuleDef = PyModuleDef { -//! m_base: PyModuleDef_HEAD_INIT, -//! m_name: c_str!("string_sum").as_ptr(), -//! m_doc: c_str!("A Python module written in Rust.").as_ptr(), -//! m_size: 0, -//! m_methods: unsafe { METHODS.as_mut_ptr().cast() }, -//! m_slots: std::ptr::null_mut(), -//! m_traverse: None, -//! m_clear: None, -//! m_free: None, -//! }; -//! -//! static mut METHODS: [PyMethodDef; 2] = [ -//! PyMethodDef { -//! ml_name: c_str!("sum_as_string").as_ptr(), -//! ml_meth: PyMethodDefPointer { -//! PyCFunctionFast: sum_as_string, -//! }, -//! ml_flags: METH_FASTCALL, -//! ml_doc: c_str!("returns the sum of two integers as a string").as_ptr(), -//! }, -//! // A zeroed PyMethodDef to mark the end of the array. -//! PyMethodDef::zeroed() -//! ]; -//! -//! // The module initialization function, which must be named `PyInit_`. -//! #[allow(non_snake_case)] -//! #[no_mangle] -//! pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject { -//! PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)) -//! } +//! If you need to use conditional compilation based on Python version or how +//! Python was compiled, you need to add `pyo3-build-config` as a +//! `build-dependency` in your `Cargo.toml` as in the example above and either +//! create a new `build.rs` file or modify an existing one so that +//! `pyo3_build_config::use_pyo3_cfgs()` gets called at build time: //! -//! pub unsafe extern "C" fn sum_as_string( -//! _self: *mut PyObject, -//! args: *mut *mut PyObject, -//! nargs: Py_ssize_t, -//! ) -> *mut PyObject { -//! if nargs != 2 { -//! PyErr_SetString( -//! PyExc_TypeError, -//! c_str!("sum_as_string() expected 2 positional arguments").as_ptr(), -//! ); -//! return std::ptr::null_mut(); -//! } -//! -//! let arg1 = *args; -//! if PyLong_Check(arg1) == 0 { -//! PyErr_SetString( -//! PyExc_TypeError, -//! c_str!("sum_as_string() expected an int for positional argument 1").as_ptr(), -//! ); -//! return std::ptr::null_mut(); -//! } -//! -//! let arg1 = PyLong_AsLong(arg1); -//! if !PyErr_Occurred().is_null() { -//! return ptr::null_mut(); -//! } -//! -//! let arg2 = *args.add(1); -//! if PyLong_Check(arg2) == 0 { -//! PyErr_SetString( -//! PyExc_TypeError, -//! c_str!("sum_as_string() expected an int for positional argument 2").as_ptr(), -//! ); -//! return std::ptr::null_mut(); -//! } -//! -//! let arg2 = PyLong_AsLong(arg2); -//! if !PyErr_Occurred().is_null() { -//! return ptr::null_mut(); -//! } -//! -//! match arg1.checked_add(arg2) { -//! Some(sum) => { -//! let string = sum.to_string(); -//! PyUnicode_FromStringAndSize(string.as_ptr().cast::(), string.len() as isize) -//! } -//! None => { -//! PyErr_SetString( -//! PyExc_OverflowError, -//! c_str!("arguments too large to add").as_ptr(), -//! ); -//! std::ptr::null_mut() -//! } -//! } +//! **`build.rs`** +//! ```rust,ignore +//! fn main() { +//! pyo3_build_config::use_pyo3_cfgs() //! } //! ``` //! @@ -201,6 +152,12 @@ //! [manually][manual_builds]. Both offer more flexibility than `maturin` but require further //! configuration. //! +//! This example stores the module definition statically and uses the `PyModule_Create` function +//! in the CPython C API to register the module. This is the "old" style for registering modules +//! and has the limitation that it cannot support subinterpreters. You can also create a module +//! using the new multi-phase initialization API that does support subinterpreters. See the +//! `sequential` project located in the `examples` directory at the root of the `pyo3-ffi` crate +//! for a worked example of how to this using `pyo3-ffi`. //! //! # Using Python from Rust //! @@ -226,7 +183,7 @@ #![doc = concat!("[manual_builds]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/building-and-distribution.html#manual-builds \"Manual builds - Building and Distribution - PyO3 user guide\"")] //! [setuptools-rust]: https://github.com/PyO3/setuptools-rust "Setuptools plugin for Rust extensions" //! [PEP 384]: https://www.python.org/dev/peps/pep-0384 "PEP 384 -- Defining a Stable ABI" -#![doc = concat!("[Features chapter of the guide]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/features.html#features-reference \"Features Reference - PyO3 user guide\"")] +#![doc = concat!("[Features chapter of the guide]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/features.html#features-reference \"Features eference - PyO3 user guide\"")] #![allow( missing_docs, non_camel_case_types, diff --git a/include/pyo3/pyo3-ffi/src/listobject.rs b/include/pyo3/pyo3-ffi/src/listobject.rs index 9d8b7ed6..881a8a87 100644 --- a/include/pyo3/pyo3-ffi/src/listobject.rs +++ b/include/pyo3/pyo3-ffi/src/listobject.rs @@ -50,6 +50,10 @@ extern "C" { arg3: Py_ssize_t, arg4: *mut PyObject, ) -> c_int; + #[cfg(Py_3_13)] + pub fn PyList_Extend(list: *mut PyObject, iterable: *mut PyObject) -> c_int; + #[cfg(Py_3_13)] + pub fn PyList_Clear(list: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyList_Sort")] pub fn PyList_Sort(arg1: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyList_Reverse")] diff --git a/include/pyo3/pyo3-ffi/src/methodobject.rs b/include/pyo3/pyo3-ffi/src/methodobject.rs index bd214409..37e1e206 100644 --- a/include/pyo3/pyo3-ffi/src/methodobject.rs +++ b/include/pyo3/pyo3-ffi/src/methodobject.rs @@ -50,7 +50,7 @@ pub type PyCFunctionFast = unsafe extern "C" fn( ) -> *mut PyObject; #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] -#[deprecated(note = "renamed to `PyCFunctionFast`")] +#[cfg_attr(Py_3_10, deprecated(note = "renamed to `PyCFunctionFast`"))] pub type _PyCFunctionFast = PyCFunctionFast; pub type PyCFunctionWithKeywords = unsafe extern "C" fn( @@ -68,6 +68,7 @@ pub type PyCFunctionFastWithKeywords = unsafe extern "C" fn( ) -> *mut PyObject; #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +#[cfg_attr(Py_3_10, deprecated(note = "renamed to `PyCFunctionFastWithKeywords`"))] pub type _PyCFunctionFastWithKeywords = PyCFunctionFastWithKeywords; #[cfg(all(Py_3_9, not(Py_LIMITED_API)))] @@ -152,7 +153,7 @@ pub union PyMethodDefPointer { /// This variant corresponds with [`METH_FASTCALL`]. #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] - #[deprecated(note = "renamed to `PyCFunctionFast`")] + #[cfg_attr(Py_3_10, deprecated(note = "renamed to `PyCFunctionFast`"))] pub _PyCFunctionFast: PyCFunctionFast, /// This variant corresponds with [`METH_FASTCALL`]. @@ -161,6 +162,7 @@ pub union PyMethodDefPointer { /// This variant corresponds with [`METH_FASTCALL`] | [`METH_KEYWORDS`]. #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + #[cfg_attr(Py_3_10, deprecated(note = "renamed to `PyCFunctionFastWithKeywords`"))] pub _PyCFunctionFastWithKeywords: PyCFunctionFastWithKeywords, /// This variant corresponds with [`METH_FASTCALL`] | [`METH_KEYWORDS`]. diff --git a/include/pyo3/pyo3-ffi/src/moduleobject.rs b/include/pyo3/pyo3-ffi/src/moduleobject.rs index ff6458f4..2417664a 100644 --- a/include/pyo3/pyo3-ffi/src/moduleobject.rs +++ b/include/pyo3/pyo3-ffi/src/moduleobject.rs @@ -88,6 +88,10 @@ pub const Py_mod_create: c_int = 1; pub const Py_mod_exec: c_int = 2; #[cfg(Py_3_12)] pub const Py_mod_multiple_interpreters: c_int = 3; +#[cfg(Py_3_13)] +pub const Py_mod_gil: c_int = 4; + +// skipped private _Py_mod_LAST_SLOT #[cfg(Py_3_12)] pub const Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED: *mut c_void = 0 as *mut c_void; @@ -96,7 +100,15 @@ pub const Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED: *mut c_void = 1 as *mut c_void #[cfg(Py_3_12)] pub const Py_MOD_PER_INTERPRETER_GIL_SUPPORTED: *mut c_void = 2 as *mut c_void; -// skipped non-limited _Py_mod_LAST_SLOT +#[cfg(Py_3_13)] +pub const Py_MOD_GIL_USED: *mut c_void = 0 as *mut c_void; +#[cfg(Py_3_13)] +pub const Py_MOD_GIL_NOT_USED: *mut c_void = 1 as *mut c_void; + +#[cfg(all(not(Py_LIMITED_API), Py_GIL_DISABLED))] +extern "C" { + pub fn PyUnstable_Module_SetGIL(module: *mut PyObject, gil: *mut c_void) -> c_int; +} #[repr(C)] pub struct PyModuleDef { diff --git a/integration/requirements.txt b/integration/requirements.txt index e5b290de..2f47d3a7 100644 --- a/integration/requirements.txt +++ b/integration/requirements.txt @@ -1,3 +1,3 @@ flask;sys_platform!="win" gunicorn;sys_platform!="win" -httpx==0.24.1;sys_platform!="win" +httpx==0.27.2;sys_platform!="win" diff --git a/requirements.txt b/requirements.txt index 9981cc8c..d211c555 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ -r test/requirements.txt maturin mypy==1.13.0 -ruff==0.7.1 +ruff==0.7.3 diff --git a/script/install-fedora b/script/install-fedora index 19a5a455..6d14860e 100755 --- a/script/install-fedora +++ b/script/install-fedora @@ -14,7 +14,7 @@ export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}" rm /etc/yum.repos.d/fedora-cisco-openh264.repo || true -dnf install --setopt=install_weak_deps=false -y rustup clang lld "${PYTHON_PACKAGE}" +dnf install --setopt=install_weak_deps=false -y rustup clang lld "${PYTHON_PACKAGE}" python3-uv rustup-init --default-toolchain "${RUST_TOOLCHAIN}-${TARGET}" --profile minimal --component rust-src -y source "${HOME}/.cargo/env" @@ -24,7 +24,6 @@ cp ci/config.toml .cargo/config.toml cargo fetch --target="${TARGET}" & -curl -LsSf https://astral.sh/uv/install.sh | sh rm -rf "${VENV}" uv venv --python "${PYTHON}" "${VENV}" source "${VENV}/bin/activate" diff --git a/script/lint b/script/lint index 22f5742e..94a46b1c 100755 --- a/script/lint +++ b/script/lint @@ -2,7 +2,7 @@ set -eou pipefail -to_lint="./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py script/pydataclass script/pymem +to_lint="./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py script/pydataclass script/pysort script/pynumpy script/pynonstr script/pycorrectness script/graph integration/init integration/wsgi.py integration/typestubs.py integration/thread" diff --git a/script/pymem b/script/pymem deleted file mode 100755 index 6b97bb0d..00000000 --- a/script/pymem +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import io -import subprocess - -from tabulate import tabulate - -buf = io.StringIO() - -headers = ("Library", "import, read() RSS (MiB)", "loads() increase in RSS (MiB)") - -LIBRARIES = ("orjson", "ujson", "rapidjson", "simplejson", "json") - -FIXTURES = ("canada.json", "citm_catalog.json", "github.json", "twitter.json") - -for fixture in sorted(FIXTURES, reverse=True): - table = [] - buf.write("\n" + "#### " + fixture + "\n\n") - for lib_name in LIBRARIES: - proc = subprocess.Popen( - ("bench/run_mem", f"data/{fixture}.xz", lib_name), stdout=subprocess.PIPE - ) - output = proc.stdout.readline().decode("utf-8").strip().split(",") - mem_base = int(output[0]) / 1024 / 1024 - mem_diff = int(output[1]) / 1024 / 1024 - correct = bool(int(output[2])) - if correct: - table.append((lib_name, f"{mem_base:,.1f}", f"{mem_diff:,.1f}")) - else: - table.append((lib_name, "", "")) - buf.write(tabulate(table, headers, tablefmt="github") + "\n") - -print(buf.getvalue()) diff --git a/src/lib.rs b/src/lib.rs index 64ace9e1..e65cd7f5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ #![cfg_attr(feature = "intrinsics", feature(core_intrinsics))] #![cfg_attr(feature = "optimize", feature(optimize_attribute))] #![cfg_attr(feature = "unstable-simd", feature(portable_simd))] +#![allow(unknown_lints)] #![allow(internal_features)] // core_intrinsics #![allow(non_camel_case_types)] #![allow(static_mut_refs)] diff --git a/src/util.rs b/src/util.rs index 8add0f75..2958d287 100644 --- a/src/util.rs +++ b/src/util.rs @@ -267,7 +267,11 @@ macro_rules! reserve_pretty { macro_rules! assume { ($expr:expr) => { debug_assert!($expr); - #[cfg(feature = "intrinsics")] + #[cfg(feature = "assert_unchecked")] + unsafe { + core::hint::assert_unchecked($expr); + }; + #[cfg(all(not(feature = "assert_unchecked"), feature = "intrinsics"))] unsafe { core::intrinsics::assume($expr); }; diff --git a/test/requirements.txt b/test/requirements.txt index 2c04102f..f2791df7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -7,4 +7,3 @@ psutil;(sys_platform=="linux" or sys_platform == "macos") and platform_machine== pytest pytz typing_extensions;python_version<"3.8" -xxhash==1.4.3;sys_platform=="linux" and platform_machine=="x86_64" and python_version<"3.9" # creates non-compact ASCII for test_str_ascii diff --git a/test/test_dict.py b/test/test_dict.py index 82468f31..b02c9e31 100644 --- a/test/test_dict.py +++ b/test/test_dict.py @@ -1,9 +1,14 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import sys +import sysconfig + import pytest import orjson +IS_OLD_WIN_32 = sys.version_info.minor < 11 and sysconfig.get_platform() == "win32" + class TestDict: def test_dict(self): @@ -126,6 +131,7 @@ def test_in_place(self): data["id"] = "new" assert orjson.dumps(data) == b'{"id":"new","static":"msg"}' + @pytest.mark.skipif(IS_OLD_WIN_32, reason="Takes ages on GitHub Actions") def test_dict_0xff(self): "dk_size <= 0xff" data = {str(idx): idx for idx in range(0, 0xFF)} @@ -134,6 +140,7 @@ def test_dict_0xff(self): data["113"] = 2 assert orjson.loads(orjson.dumps(data)) == data + @pytest.mark.skipif(IS_OLD_WIN_32, reason="Takes ages on GitHub Actions") def test_dict_0xff_repeated(self): "dk_size <= 0xff repeated" for _ in range(0, 100): @@ -143,6 +150,7 @@ def test_dict_0xff_repeated(self): data["113"] = 2 assert orjson.loads(orjson.dumps(data)) == data + @pytest.mark.skipif(IS_OLD_WIN_32, reason="Takes ages on GitHub Actions") def test_dict_0xffff(self): "dk_size <= 0xffff" data = {str(idx): idx for idx in range(0, 0xFFFF)} @@ -151,6 +159,7 @@ def test_dict_0xffff(self): data["113"] = 2 assert orjson.loads(orjson.dumps(data)) == data + @pytest.mark.skipif(IS_OLD_WIN_32, reason="Takes ages on GitHub Actions") def test_dict_0xffff_repeated(self): "dk_size <= 0xffff repeated" for _ in range(0, 100): diff --git a/test/test_type.py b/test/test_type.py index 81c89307..3a3f7a56 100644 --- a/test/test_type.py +++ b/test/test_type.py @@ -5,11 +5,6 @@ import pytest -try: - import xxhash -except ImportError: - xxhash = None - import orjson @@ -260,17 +255,6 @@ def test_str_surrogates_dumps(self): orjson.JSONEncodeError, orjson.dumps, b"\xed\xa0\xbd\xed\xba\x80" ) # \ud83d\ude80 - @pytest.mark.skipif( - xxhash is None, reason="xxhash install broken on win, python3.9, Azure" - ) - def test_str_ascii(self): - """ - str is ASCII but not compact - """ - digest = xxhash.xxh32_hexdigest("12345") - for _ in range(2): - assert orjson.dumps(digest) == b'"b30d56b4"' - def test_bytes_dumps(self): """ bytes dumps not supported