Skip to content

Commit

Permalink
fix(cmake): improved cross-compilation support (#5083)
Browse files Browse the repository at this point in the history
* fix(cmake): do not use Python::Interpreter when cross-compiling

* chore: apply cmake-format to pybind11NewTools.cmake

* fix(cmake): do not look for Python Interpreter component when cross-compiling

* feat(cmake): guess Python extension suffix

* fix: add pybind11GuessPythonExtSuffix.cmake to packaging test

* Use PYBIND11_CROSSCOMPILING instead of CMAKE_CROSSCOMPILING

* refactor: require PYBIND11_USE_CROSSCOMPILING

Signed-off-by: Henry Schreiner <[email protected]>

---------

Signed-off-by: Henry Schreiner <[email protected]>
Co-authored-by: Henry Schreiner <[email protected]>
  • Loading branch information
tttapa and henryiii authored Jun 6, 2024
1 parent b9794be commit 9b3a200
Show file tree
Hide file tree
Showing 7 changed files with 352 additions and 62 deletions.
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ option(PYBIND11_NUMPY_1_ONLY
set(PYBIND11_INTERNALS_VERSION
""
CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
option(PYBIND11_USE_CROSSCOMPILING "Respect CMAKE_CROSSCOMPILING" OFF)

if(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
add_compile_definitions(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
Expand Down Expand Up @@ -299,6 +300,7 @@ if(PYBIND11_INSTALL)
tools/pybind11Common.cmake
tools/pybind11Tools.cmake
tools/pybind11NewTools.cmake
tools/pybind11GuessPythonExtSuffix.cmake
DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})

if(NOT PYBIND11_EXPORT_NAME)
Expand Down
1 change: 1 addition & 0 deletions tests/extra_python_package/test_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
"share/cmake/pybind11/pybind11Common.cmake",
"share/cmake/pybind11/pybind11Config.cmake",
"share/cmake/pybind11/pybind11ConfigVersion.cmake",
"share/cmake/pybind11/pybind11GuessPythonExtSuffix.cmake",
"share/cmake/pybind11/pybind11NewTools.cmake",
"share/cmake/pybind11/pybind11Targets.cmake",
"share/cmake/pybind11/pybind11Tools.cmake",
Expand Down
2 changes: 1 addition & 1 deletion tools/FindPythonLibsNew.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ endif()
# Make sure the Python has the same pointer-size as the chosen compiler
# Skip if CMAKE_SIZEOF_VOID_P is not defined
# This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN)
if(NOT CMAKE_CROSSCOMPILING
if(NOT _PYBIND11_CROSSCOMPILING
AND CMAKE_SIZEOF_VOID_P
AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}"))
if(PythonLibsNew_FIND_REQUIRED)
Expand Down
12 changes: 11 additions & 1 deletion tools/pybind11Common.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,16 @@ set(pybind11_INCLUDE_DIRS
"${pybind11_INCLUDE_DIR}"
CACHE INTERNAL "Include directory for pybind11 (Python not requested)")

if(CMAKE_CROSSCOMPILING AND PYBIND11_USE_CROSSCOMPILING)
set(_PYBIND11_CROSSCOMPILING
ON
CACHE INTERNAL "")
else()
set(_PYBIND11_CROSSCOMPILING
OFF
CACHE INTERNAL "")
endif()

# --------------------- Shared targets ----------------------------

# Build an interface library target:
Expand Down Expand Up @@ -195,7 +205,7 @@ endif()

# --------------------- pybind11_find_import -------------------------------

if(NOT _pybind11_nopython)
if(NOT _pybind11_nopython AND NOT _PYBIND11_CROSSCOMPILING)
# Check to see if modules are importable. Use REQUIRED to force an error if
# one of the modules is not found. <package_name>_FOUND will be set if the
# package was found (underscores replace dashes if present). QUIET will hide
Expand Down
86 changes: 86 additions & 0 deletions tools/pybind11GuessPythonExtSuffix.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
cmake_minimum_required(VERSION 3.5)

function(pybind11_guess_python_module_extension python)

# The SETUPTOOLS_EXT_SUFFIX environment variable takes precedence:
if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ENV{SETUPTOOLS_EXT_SUFFIX})
message(
STATUS
"Getting Python extension suffix from ENV{SETUPTOOLS_EXT_SUFFIX}: $ENV{SETUPTOOLS_EXT_SUFFIX}"
)
set(PYTHON_MODULE_EXT_SUFFIX
"$ENV{SETUPTOOLS_EXT_SUFFIX}"
CACHE
STRING
"Extension suffix for Python extension modules (Initialized from SETUPTOOLS_EXT_SUFFIX)")
endif()
# If that didn't work, use the Python_SOABI variable:
if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ${python}_SOABI)
message(
STATUS "Determining Python extension suffix based on ${python}_SOABI: ${${python}_SOABI}")
# The final extension depends on the system
set(_PY_BUILD_EXTENSION "${CMAKE_SHARED_MODULE_SUFFIX}")
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
set(_PY_BUILD_EXTENSION ".pyd")
endif()
# If the SOABI already has an extension, use it as the full suffix
# (used for debug versions of Python on Windows)
if(${python}_SOABI MATCHES "\\.")
set(PYTHON_MODULE_EXT_SUFFIX "${${python}_SOABI}")
# If the SOABI is empty, this is usually a bug, but we generate a
# correct extension anyway, which is the best we can do
elseif("${${python}_SOABI}" STREQUAL "")
message(
WARNING
"${python}_SOABI is defined but empty. You may want to set PYTHON_MODULE_EXT_SUFFIX explicitly."
)
set(PYTHON_MODULE_EXT_SUFFIX "${_PY_BUILD_EXTENSION}")
# Otherwise, add the system-dependent extension to it
else()
set(PYTHON_MODULE_EXT_SUFFIX ".${${python}_SOABI}${_PY_BUILD_EXTENSION}")
endif()
endif()

# If we could not deduce the extension suffix, unset the results:
if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_DEBUG_POSTFIX PARENT_SCOPE)
unset(PYTHON_MODULE_EXTENSION PARENT_SCOPE)
unset(PYTHON_IS_DEBUG PARENT_SCOPE)
return()
endif()

# Sanity checks:
if(${python}_SOABI AND NOT (PYTHON_MODULE_EXT_SUFFIX STREQUAL ${python}_SOABI
OR PYTHON_MODULE_EXT_SUFFIX MATCHES "\\.${${python}_SOABI}\\."))
message(
WARNING
"Python extension suffix (${PYTHON_MODULE_EXT_SUFFIX}) does not match ${python}_SOABI (${${python}_SOABI})."
)
endif()

# Separate file name postfix from extension: (https://github.com/pybind/pybind11/issues/4699)
get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
get_filename_component(_PYTHON_MODULE_EXTENSION "${PYTHON_MODULE_EXT_SUFFIX}" EXT)

# Try to deduce the debug ABI from the extension suffix:
if(NOT DEFINED _PYTHON_IS_DEBUG)
if(_PYTHON_MODULE_EXTENSION MATCHES "^\\.(cpython-|cp|pypy)[0-9]+dm?-"
OR _PYTHON_MODULE_DEBUG_POSTFIX MATCHES "^_d")
set(_PYTHON_IS_DEBUG On)
else()
set(_PYTHON_IS_DEBUG Off)
endif()
endif()

# Return results
set(PYTHON_MODULE_DEBUG_POSTFIX
"${_PYTHON_MODULE_DEBUG_POSTFIX}"
PARENT_SCOPE)
set(PYTHON_MODULE_EXTENSION
"${_PYTHON_MODULE_EXTENSION}"
PARENT_SCOPE)
set(PYTHON_IS_DEBUG
"${_PYTHON_IS_DEBUG}"
PARENT_SCOPE)

endfunction()
150 changes: 90 additions & 60 deletions tools/pybind11NewTools.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,13 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
set(Python_ROOT_DIR "$ENV{pythonLocation}")
endif()

# Interpreter should not be found when cross-compiling
if(_PYBIND11_CROSSCOMPILING)
set(_pybind11_interp_component "")
else()
set(_pybind11_interp_component Interpreter)
endif()

# Development.Module support (required for manylinux) started in 3.18
if(CMAKE_VERSION VERSION_LESS 3.18)
set(_pybind11_dev_component Development)
Expand All @@ -48,8 +55,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
endif()
endif()

find_package(Python 3.6 REQUIRED COMPONENTS Interpreter ${_pybind11_dev_component}
${_pybind11_quiet} ${_pybind11_global_keyword})
find_package(
Python 3.6 REQUIRED COMPONENTS ${_pybind11_interp_component} ${_pybind11_dev_component}
${_pybind11_quiet} ${_pybind11_global_keyword})

# If we are in submodule mode, export the Python targets to global targets.
# If this behavior is not desired, FindPython _before_ pybind11.
Expand All @@ -59,7 +67,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
if(TARGET Python::Python)
set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE)
endif()
set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE)
if(TARGET Python::Interpreter)
set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE)
endif()
if(TARGET Python::Module)
set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE)
endif()
Expand Down Expand Up @@ -100,69 +110,89 @@ if(PYBIND11_MASTER_PROJECT)
endif()
endif()

# If a user finds Python, they may forget to include the Interpreter component
# and the following two steps require it. It is highly recommended by CMake
# when finding development libraries anyway, so we will require it.
if(NOT DEFINED ${_Python}_EXECUTABLE)
message(
FATAL_ERROR
"${_Python} was found without the Interpreter component. Pybind11 requires this component.")

endif()

if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL
PYBIND11_PYTHON_EXECUTABLE_LAST)
# Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed
unset(PYTHON_IS_DEBUG CACHE)
unset(PYTHON_MODULE_EXTENSION CACHE)
endif()

set(PYBIND11_PYTHON_EXECUTABLE_LAST
"${${_Python}_EXECUTABLE}"
CACHE INTERNAL "Python executable during the last CMake run")

if(NOT DEFINED PYTHON_IS_DEBUG)
# Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter
execute_process(
COMMAND "${${_Python}_EXECUTABLE}" "-c"
"import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))"
RESULT_VARIABLE _PYTHON_IS_DEBUG)
set(PYTHON_IS_DEBUG
"${_PYTHON_IS_DEBUG}"
CACHE INTERNAL "Python debug status")
endif()

# Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
# required for PyPy3 (as of 7.3.1)
if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
execute_process(
COMMAND
"${${_Python}_EXECUTABLE}" "-c"
"import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))"
OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX
ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR
OUTPUT_STRIP_TRAILING_WHITESPACE)

if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "")
if(NOT _PYBIND11_CROSSCOMPILING)
# If a user finds Python, they may forget to include the Interpreter component
# and the following two steps require it. It is highly recommended by CMake
# when finding development libraries anyway, so we will require it.
if(NOT DEFINED ${_Python}_EXECUTABLE)
message(
FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'"
"package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}"
FATAL_ERROR
"${_Python} was found without the Interpreter component. Pybind11 requires this component."
)

endif()

# This needs to be available for the pybind11_extension function
if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
set(PYTHON_MODULE_DEBUG_POSTFIX
"${_PYTHON_MODULE_DEBUG_POSTFIX}"
CACHE INTERNAL "")
if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL
PYBIND11_PYTHON_EXECUTABLE_LAST)
# Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed
unset(PYTHON_IS_DEBUG CACHE)
unset(PYTHON_MODULE_EXTENSION CACHE)
endif()

if(NOT DEFINED PYTHON_MODULE_EXTENSION)
get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
set(PYTHON_MODULE_EXTENSION
"${_PYTHON_MODULE_EXTENSION}"
CACHE INTERNAL "")
set(PYBIND11_PYTHON_EXECUTABLE_LAST
"${${_Python}_EXECUTABLE}"
CACHE INTERNAL "Python executable during the last CMake run")

if(NOT DEFINED PYTHON_IS_DEBUG)
# Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter
execute_process(
COMMAND "${${_Python}_EXECUTABLE}" "-c"
"import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))"
RESULT_VARIABLE _PYTHON_IS_DEBUG)
set(PYTHON_IS_DEBUG
"${_PYTHON_IS_DEBUG}"
CACHE INTERNAL "Python debug status")
endif()

# Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
# required for PyPy3 (as of 7.3.1)
if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
execute_process(
COMMAND
"${${_Python}_EXECUTABLE}" "-c"
"import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))"
OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX
ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR
OUTPUT_STRIP_TRAILING_WHITESPACE)

if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "")
message(
FATAL_ERROR
"pybind11 could not query the module file extension, likely the 'distutils'"
"package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}")
endif()

# This needs to be available for the pybind11_extension function
if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
set(PYTHON_MODULE_DEBUG_POSTFIX
"${_PYTHON_MODULE_DEBUG_POSTFIX}"
CACHE INTERNAL "")
endif()

if(NOT DEFINED PYTHON_MODULE_EXTENSION)
get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
set(PYTHON_MODULE_EXTENSION
"${_PYTHON_MODULE_EXTENSION}"
CACHE INTERNAL "")
endif()
endif()
else()
if(NOT DEFINED PYTHON_IS_DEBUG
OR NOT DEFINED PYTHON_MODULE_EXTENSION
OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake")
pybind11_guess_python_module_extension("${_Python}")
endif()
# When cross-compiling, we cannot query the Python interpreter, so we require
# the user to set these variables explicitly.
if(NOT DEFINED PYTHON_IS_DEBUG
OR NOT DEFINED PYTHON_MODULE_EXTENSION
OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
message(
FATAL_ERROR
"When cross-compiling, you should set the PYTHON_IS_DEBUG, PYTHON_MODULE_EXTENSION and PYTHON_MODULE_DEBUG_POSTFIX \
variables appropriately before loading pybind11 (e.g. in your CMake toolchain file)")
endif()
endif()

Expand Down
Loading

0 comments on commit 9b3a200

Please sign in to comment.