From 53edf889921859c820bccfae5fe6a291ec9350bb Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 7 Nov 2024 15:28:46 -0700 Subject: [PATCH 1/7] use the correct library name for the free-threaded build --- pyo3-build-config/src/impl_.rs | 84 ++++++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index ec65259115f..a96136c6b1f 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -248,6 +248,7 @@ print("executable", sys.executable) print("calcsize_pointer", struct.calcsize("P")) print("mingw", get_platform().startswith("mingw")) print("ext_suffix", get_config_var("EXT_SUFFIX")) +print("gil_disabled", get_config_var("Py_GIL_DISABLED")) "#; let output = run_python_script(interpreter.as_ref(), SCRIPT)?; let map: HashMap = parse_script_output(&output); @@ -290,6 +291,13 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let implementation = map["implementation"].parse()?; + let gil_disabled = match map["gil_disabled"].as_str() { + "1" => true, + "0" => false, + "None" => false, + _ => panic!("Unknown Py_GIL_DISABLED value"), + }; + let lib_name = if cfg!(windows) { default_lib_name_windows( version, @@ -300,12 +308,14 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) // on Windows from sysconfig - e.g. ext_suffix may be // `_d.cp312-win_amd64.pyd` for 3.12 debug build map["ext_suffix"].starts_with("_d."), + gil_disabled, ) } else { default_lib_name_unix( version, implementation, map.get("ld_version").map(String::as_str), + gil_disabled, ) }; @@ -375,10 +385,21 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) _ => false, }; let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string); + let gil_disabled = match sysconfigdata.get_value("Py_GIL_DISABLED") { + Some(value) => { + if value == "1" { + true + } else { + false + } + } + None => false, + }; let lib_name = Some(default_lib_name_unix( version, implementation, sysconfigdata.get_value("LDVERSION"), + gil_disabled, )); let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P") .map(|bytes_width: u32| bytes_width * 8) @@ -1528,6 +1549,7 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf abi3, false, false, + false, )) } else { None @@ -1604,9 +1626,10 @@ fn default_lib_name_for_target( abi3, false, false, + false, )) } else if is_linking_libpython_for_target(target) { - Some(default_lib_name_unix(version, implementation, None)) + Some(default_lib_name_unix(version, implementation, None, false)) } else { None } @@ -1618,16 +1641,26 @@ fn default_lib_name_windows( abi3: bool, mingw: bool, debug: bool, + gil_disabled: bool, ) -> String { if debug { // CPython bug: linking against python3_d.dll raises error // https://github.com/python/cpython/issues/101614 - format!("python{}{}_d", version.major, version.minor) + if gil_disabled { + format!("python{}{}t_d", version.major, version.minor) + } else { + format!("python{}{}_d", version.major, version.minor) + } } else if abi3 && !(implementation.is_pypy() || implementation.is_graalpy()) { WINDOWS_ABI3_LIB_NAME.to_owned() } else if mingw { + if gil_disabled { + panic!("MinGW free-threaded builds are not currently tested or supported") + } // https://packages.msys2.org/base/mingw-w64-python format!("python{}.{}", version.major, version.minor) + } else if gil_disabled { + format!("python{}{}t", version.major, version.minor) } else { format!("python{}{}", version.major, version.minor) } @@ -1637,6 +1670,7 @@ fn default_lib_name_unix( version: PythonVersion, implementation: PythonImplementation, ld_version: Option<&str>, + gil_disabled: bool, ) -> String { match implementation { PythonImplementation::CPython => match ld_version { @@ -1644,7 +1678,11 @@ fn default_lib_name_unix( None => { if version > PythonVersion::PY37 { // PEP 3149 ABI version tags are finally gone - format!("python{}.{}", version.major, version.minor) + if gil_disabled { + format!("python{}.{}t", version.major, version.minor) + } else { + format!("python{}.{}", version.major, version.minor) + } } else { // Work around https://bugs.python.org/issue36707 format!("python{}.{}m", version.major, version.minor) @@ -2351,6 +2389,7 @@ mod tests { false, false, false, + false, ), "python39", ); @@ -2361,6 +2400,7 @@ mod tests { true, false, false, + false, ), "python3", ); @@ -2371,6 +2411,7 @@ mod tests { false, true, false, + false, ), "python3.9", ); @@ -2381,6 +2422,7 @@ mod tests { true, true, false, + false, ), "python3", ); @@ -2391,6 +2433,7 @@ mod tests { true, false, false, + false, ), "python39", ); @@ -2401,6 +2444,7 @@ mod tests { false, false, true, + false, ), "python39_d", ); @@ -2413,6 +2457,7 @@ mod tests { true, false, true, + false, ), "python39_d", ); @@ -2423,16 +2468,31 @@ mod tests { use PythonImplementation::*; // Defaults to python3.7m for CPython 3.7 assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 7 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 7 }, + CPython, + None, + false + ), "python3.7m", ); // Defaults to pythonX.Y for CPython 3.8+ assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 8 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 8 }, + CPython, + None, + false + ), "python3.8", ); assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + CPython, + None, + false + ), "python3.9", ); // Can use ldversion to override for CPython @@ -2440,19 +2500,25 @@ mod tests { super::default_lib_name_unix( PythonVersion { major: 3, minor: 9 }, CPython, - Some("3.7md") + Some("3.7md"), + false ), "python3.7md", ); // PyPy 3.9 includes ldversion assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, None), + super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, None, false), "pypy3.9-c", ); assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, Some("3.9d")), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + PyPy, + Some("3.9d"), + false + ), "pypy3.9d-c", ); } From 551d4b4565ba625b5aa32826ef32fcb162893bcb Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 8 Nov 2024 12:19:25 -0700 Subject: [PATCH 2/7] Query sysconfig on windows for Python 3.13 and newer --- pyo3-build-config/src/impl_.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index a96136c6b1f..a473f888579 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1127,9 +1127,13 @@ impl BuildFlags { /// the interpreter and printing variables of interest from /// sysconfig.get_config_vars. fn from_interpreter(interpreter: impl AsRef) -> Result { - // sysconfig is missing all the flags on windows, so we can't actually - // query the interpreter directly for its build flags. - if cfg!(windows) { + let script = String::from("import sys;print(sys.version_info < (3, 13))"); + let stdout = run_python_script(interpreter.as_ref(), &script)?; + + // sysconfig is missing all the flags on windows for Python 3.12 and + // older, so we can't actually query the interpreter directly for its + // build flags on those versions. + if cfg!(windows) && stdout.trim_end() == "True" { return Ok(Self::new()); } From aff79069db29bb479f289dbd3651661c97cab963 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 8 Nov 2024 13:52:42 -0700 Subject: [PATCH 3/7] attempt to fix freethreaded python pyo3-ffi-check on windows --- pyo3-ffi-check/build.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pyo3-ffi-check/build.rs b/pyo3-ffi-check/build.rs index ca4a17b6a61..0ae7d4218dd 100644 --- a/pyo3-ffi-check/build.rs +++ b/pyo3-ffi-check/build.rs @@ -8,12 +8,28 @@ fn main() { "import sysconfig; print(sysconfig.get_config_var('INCLUDEPY'), end='');", ) .expect("failed to get lib dir"); + let gil_disabled_on_windows = dbg!(config + .run_python_script( + "import sysconfig; import platform; print(sysconfig.get_config_var('Py_GIL_DISABLED') == 1 and platform.system() == 'Windows');", + ) + .expect("failed to get Py_GIL_DISABLED").trim_end()) == "True"; + + let clang_args = if gil_disabled_on_windows { + vec![ + format!("-I{python_include_dir}"), + "-DPy_GIL_DISABLED".to_string(), + ] + } else { + vec![format!("-I{python_include_dir}")] + }; + + dbg!(&clang_args); println!("cargo:rerun-if-changed=wrapper.h"); let bindings = bindgen::Builder::default() .header("wrapper.h") - .clang_arg(format!("-I{python_include_dir}")) + .clang_args(clang_args) .parse_callbacks(Box::new(bindgen::CargoCallbacks)) // blocklist some values which apparently have conflicting definitions on unix .blocklist_item("FP_NORMAL") From 3f3ddd2bc44bc24384ee0f1baaa7642f3a3d184e Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 8 Nov 2024 13:52:56 -0700 Subject: [PATCH 4/7] fix clippy lint --- pyo3-build-config/src/impl_.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index a473f888579..e57204f2e3c 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -386,13 +386,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) }; let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string); let gil_disabled = match sysconfigdata.get_value("Py_GIL_DISABLED") { - Some(value) => { - if value == "1" { - true - } else { - false - } - } + Some(value) => value == "1", None => false, }; let lib_name = Some(default_lib_name_unix( From c9a1c6decf377fc40b0961b5c40754d799dc1959 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 8 Nov 2024 13:53:08 -0700 Subject: [PATCH 5/7] turn on windows free-threaded CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 220738168f8..af1217b31fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -556,7 +556,7 @@ jobs: runs-on: ${{ matrix.runner }} strategy: matrix: - runner: ["ubuntu-latest", "macos-latest"] + runner: ["ubuntu-latest", "macos-latest", "windows-latest"] steps: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 From bd9958476a01276b5c6b3796e2dcd7736993d990 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 8 Nov 2024 14:19:05 -0700 Subject: [PATCH 6/7] only check python version on windows --- pyo3-build-config/src/impl_.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index e57204f2e3c..6d2326429d2 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1121,14 +1121,15 @@ impl BuildFlags { /// the interpreter and printing variables of interest from /// sysconfig.get_config_vars. fn from_interpreter(interpreter: impl AsRef) -> Result { - let script = String::from("import sys;print(sys.version_info < (3, 13))"); - let stdout = run_python_script(interpreter.as_ref(), &script)?; - // sysconfig is missing all the flags on windows for Python 3.12 and // older, so we can't actually query the interpreter directly for its // build flags on those versions. - if cfg!(windows) && stdout.trim_end() == "True" { - return Ok(Self::new()); + if cfg!(windows) { + let script = String::from("import sys;print(sys.version_info < (3, 13))"); + let stdout = run_python_script(interpreter.as_ref(), &script)?; + if stdout.trim_end() == "True" { + return Ok(Self::new()); + } } let mut script = String::from("import sysconfig\n"); From 64a15da55102be41abc415558ed1ed86a5a2f775 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 8 Nov 2024 14:23:56 -0700 Subject: [PATCH 7/7] remove debug prints --- pyo3-ffi-check/build.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyo3-ffi-check/build.rs b/pyo3-ffi-check/build.rs index 0ae7d4218dd..072e8c072b0 100644 --- a/pyo3-ffi-check/build.rs +++ b/pyo3-ffi-check/build.rs @@ -8,11 +8,11 @@ fn main() { "import sysconfig; print(sysconfig.get_config_var('INCLUDEPY'), end='');", ) .expect("failed to get lib dir"); - let gil_disabled_on_windows = dbg!(config + let gil_disabled_on_windows = config .run_python_script( "import sysconfig; import platform; print(sysconfig.get_config_var('Py_GIL_DISABLED') == 1 and platform.system() == 'Windows');", ) - .expect("failed to get Py_GIL_DISABLED").trim_end()) == "True"; + .expect("failed to get Py_GIL_DISABLED").trim_end() == "True"; let clang_args = if gil_disabled_on_windows { vec![ @@ -23,8 +23,6 @@ fn main() { vec![format!("-I{python_include_dir}")] }; - dbg!(&clang_args); - println!("cargo:rerun-if-changed=wrapper.h"); let bindings = bindgen::Builder::default()