Add support for GraalPy (#5380)

* Initial support for GraalPy

* Mark tests that currently fail on GraalPy with xfail

* Add graalpy to CI

* Limit test deps on graalpy to available binary wheels

* Skip cmake test installed_function on GraalPy

CMake won't find libpython on GraalPy, it either fails or silently picks
CPython's libpython.

* Factor out setting function docstrings into a macro

* Try to narrow down skipped tests

.github/workflows/ci.yml

@@ -39,6 +39,7 @@ jobs:
         - 'pypy-3.8'
         - 'pypy-3.9'
         - 'pypy-3.10'
+        - 'graalpy-24.1'
 
         # Items in here will either be added to the build matrix (if not
         # present), or add new keys to an existing matrix element if all the
@@ -67,6 +68,10 @@ jobs:
           # Extra ubuntu latest job
           - runs-on: ubuntu-latest
             python: '3.11'
+        exclude:
+          # The setup-python action currently doesn't have graalpy for windows
+          - python: 'graalpy-24.1'
+            runs-on: 'windows-2022'
 
 
     name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • x64 ${{ matrix.args }}"

include/pybind11/cast.h

@@ -343,7 +343,7 @@ class type_caster<bool> {
 #else
             // Alternate approach for CPython: this does the same as the above, but optimized
             // using the CPython API so as to avoid an unneeded attribute lookup.
-            else if (auto *tp_as_number = src.ptr()->ob_type->tp_as_number) {
+            else if (auto *tp_as_number = Py_TYPE(src.ptr())->tp_as_number) {
                 if (PYBIND11_NB_BOOL(tp_as_number)) {
                     res = (*PYBIND11_NB_BOOL(tp_as_number))(src.ptr());
                 }

include/pybind11/detail/common.h

@@ -299,7 +299,7 @@ PYBIND11_WARNING_DISABLE_MSVC(4505)
 #    define PYBIND11_INTERNAL_NUMPY_1_ONLY_DETECTED
 #endif
 
-#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+#if (defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
 #    define PYBIND11_SIMPLE_GIL_MANAGEMENT
 #endif
 
@@ -387,6 +387,20 @@ PYBIND11_WARNING_POP
 #define PYBIND11_CONCAT(first, second) first##second
 #define PYBIND11_ENSURE_INTERNALS_READY pybind11::detail::get_internals();
 
+#if !defined(GRAALVM_PYTHON)
+#    define PYBIND11_PYCFUNCTION_GET_DOC(func) ((func)->m_ml->ml_doc)
+#    define PYBIND11_PYCFUNCTION_SET_DOC(func, doc)                                               \
+        do {                                                                                      \
+            (func)->m_ml->ml_doc = (doc);                                                         \
+        } while (0)
+#else
+#    define PYBIND11_PYCFUNCTION_GET_DOC(func) (GraalPyCFunction_GetDoc((PyObject *) (func)))
+#    define PYBIND11_PYCFUNCTION_SET_DOC(func, doc)                                               \
+        do {                                                                                      \
+            GraalPyCFunction_SetDoc((PyObject *) (func), (doc));                                  \
+        } while (0)
+#endif
+
 #define PYBIND11_CHECK_PYTHON_VERSION                                                             \
     {                                                                                             \
         const char *compiled_ver                                                                  \

include/pybind11/detail/internals.h

@@ -454,7 +454,7 @@ inline void translate_local_exception(std::exception_ptr p) {
 
 inline object get_python_state_dict() {
     object state_dict;
-#if PYBIND11_INTERNALS_VERSION <= 4 || defined(PYPY_VERSION)
+#if PYBIND11_INTERNALS_VERSION <= 4 || defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)
     state_dict = reinterpret_borrow<object>(PyEval_GetBuiltins());
 #else
 #    if PY_VERSION_HEX < 0x03090000
@@ -727,7 +727,8 @@ const char *c_str(Args &&...args) {
 }
 
 inline const char *get_function_record_capsule_name() {
-#if PYBIND11_INTERNALS_VERSION > 4
+    // On GraalPy, pointer equality of the names is currently not guaranteed
+#if PYBIND11_INTERNALS_VERSION > 4 && !defined(GRAALVM_PYTHON)
     return get_internals().function_record_capsule_name.c_str();
 #else
     return nullptr;

include/pybind11/detail/type_caster_base.h

@@ -459,7 +459,7 @@ PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_i
 }
 
 inline PyThreadState *get_thread_state_unchecked() {
-#if defined(PYPY_VERSION)
+#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)
     return PyThreadState_GET();
 #elif PY_VERSION_HEX < 0x030D0000
     return _PyThreadState_UncheckedGet();

include/pybind11/eval.h

@@ -94,18 +94,18 @@ void exec(const char (&s)[N], object global = globals(), object local = object()
     eval<eval_statements>(s, std::move(global), std::move(local));
 }
 
-#if defined(PYPY_VERSION)
+#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)
 template <eval_mode mode = eval_statements>
 object eval_file(str, object, object) {
-    pybind11_fail("eval_file not supported in PyPy3. Use eval");
+    pybind11_fail("eval_file not supported in this interpreter. Use eval");
 }
 template <eval_mode mode = eval_statements>
 object eval_file(str, object) {
-    pybind11_fail("eval_file not supported in PyPy3. Use eval");
+    pybind11_fail("eval_file not supported in this interpreter. Use eval");
 }
 template <eval_mode mode = eval_statements>
 object eval_file(str) {
-    pybind11_fail("eval_file not supported in PyPy3. Use eval");
+    pybind11_fail("eval_file not supported in this interpreter. Use eval");
 }
 #else
 template <eval_mode mode = eval_statements>

include/pybind11/pybind11.h

@@ -573,8 +573,7 @@ class cpp_function : public function {
                 // chain.
                 chain_start = rec;
                 rec->next = chain;
-                auto rec_capsule
-                    = reinterpret_borrow<capsule>(((PyCFunctionObject *) m_ptr)->m_self);
+                auto rec_capsule = reinterpret_borrow<capsule>(PyCFunction_GET_SELF(m_ptr));
                 rec_capsule.set_pointer(unique_rec.release());
                 guarded_strdup.release();
             } else {
@@ -634,12 +633,11 @@ class cpp_function : public function {
             }
         }
 
-        /* Install docstring */
         auto *func = (PyCFunctionObject *) m_ptr;
-        std::free(const_cast<char *>(func->m_ml->ml_doc));
         // Install docstring if it's non-empty (when at least one option is enabled)
-        func->m_ml->ml_doc
-            = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str());
+        auto *doc = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str());
+        std::free(const_cast<char *>(PYBIND11_PYCFUNCTION_GET_DOC(func)));
+        PYBIND11_PYCFUNCTION_SET_DOC(func, doc);
 
         if (rec->is_method) {
             m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr());
@@ -2780,8 +2778,8 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
     }
 
     /* Don't call dispatch code if invoked from overridden function.
-       Unfortunately this doesn't work on PyPy. */
-#if !defined(PYPY_VERSION)
+       Unfortunately this doesn't work on PyPy and GraalPy. */
+#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON)
 #    if PY_VERSION_HEX >= 0x03090000
     PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get());
     if (frame != nullptr) {

include/pybind11/pytypes.h

@@ -643,7 +643,7 @@ struct error_fetch_and_normalize {
 
         bool have_trace = false;
         if (m_trace) {
-#if !defined(PYPY_VERSION)
+#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON)
             auto *tb = reinterpret_cast<PyTracebackObject *>(m_trace.ptr());
 
             // Get the deepest trace possible.
@@ -1356,7 +1356,7 @@ inline bool PyUnicode_Check_Permissive(PyObject *o) {
 #    define PYBIND11_STR_CHECK_FUN PyUnicode_Check
 #endif
 
-inline bool PyStaticMethod_Check(PyObject *o) { return o->ob_type == &PyStaticMethod_Type; }
+inline bool PyStaticMethod_Check(PyObject *o) { return Py_TYPE(o) == &PyStaticMethod_Type; }
 
 class kwargs_proxy : public handle {
 public:

tests/conftest.py

@@ -28,8 +28,8 @@
 
 @pytest.fixture(scope="session", autouse=True)
 def use_multiprocessing_forkserver_on_linux():
-    if sys.platform != "linux":
-        # The default on Windows and macOS is "spawn": If it's not broken, don't fix it.
+    if sys.platform != "linux" or sys.implementation.name == "graalpy":
+        # The default on Windows, macOS and GraalPy is "spawn": If it's not broken, don't fix it.
         return
 
     # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592

tests/env.py

@@ -12,6 +12,7 @@
 
 CPYTHON = platform.python_implementation() == "CPython"
 PYPY = platform.python_implementation() == "PyPy"
+GRAALPY = sys.implementation.name == "graalpy"
 PY_GIL_DISABLED = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
 
 

tests/requirements.txt

@@ -2,11 +2,12 @@
 build~=1.0; python_version>="3.8"
 numpy~=1.23.0; python_version=="3.8" and platform_python_implementation=="PyPy"
 numpy~=1.25.0; python_version=="3.9" and platform_python_implementation=='PyPy'
-numpy~=1.21.5; platform_python_implementation!="PyPy" and python_version>="3.8" and python_version<"3.10"
-numpy~=1.22.2; platform_python_implementation!="PyPy" and python_version=="3.10"
-numpy~=1.26.0; platform_python_implementation!="PyPy" and python_version>="3.11" and python_version<"3.13"
+numpy~=1.26.0; platform_python_implementation=="GraalVM" and sys_platform=="linux"
+numpy~=1.21.5; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.8" and python_version<"3.10"
+numpy~=1.22.2; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version=="3.10"
+numpy~=1.26.0; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.11" and python_version<"3.13"
 pytest~=7.0
 pytest-timeout
-scipy~=1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10"
-scipy~=1.8.0; platform_python_implementation!="PyPy" and python_version=="3.10" and sys_platform!='win32'
-scipy~=1.11.1; platform_python_implementation!="PyPy" and python_version>="3.11" and python_version<"3.13" and sys_platform!='win32'
+scipy~=1.5.4; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version<"3.10"
+scipy~=1.8.0; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version=="3.10" and sys_platform!='win32'
+scipy~=1.11.1; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.11" and python_version<"3.13" and sys_platform!='win32'

tests/test_buffers.py

@@ -82,6 +82,8 @@ def test_from_python():
         for j in range(m4.cols()):
             assert m3[i, j] == m4[i, j]
 
+    if env.GRAALPY:
+        pytest.skip("ConstructorStats is incompatible with GraalPy.")
     cstats = ConstructorStats.get(m.Matrix)
     assert cstats.alive() == 1
     del m3, m4
@@ -118,6 +120,8 @@ def test_to_python():
     mat2[2, 3] = 5
     assert mat2[2, 3] == 5
 
+    if env.GRAALPY:
+        pytest.skip("ConstructorStats is incompatible with GraalPy.")
     cstats = ConstructorStats.get(m.Matrix)
     assert cstats.alive() == 1
     del mat

tests/test_call_policies.cpp

@@ -95,8 +95,8 @@ TEST_SUBMODULE(call_policies, m) {
         },
         py::call_guard<DependentGuard, CustomGuard>());
 
-#if !defined(PYPY_VERSION)
-    // `py::call_guard<py::gil_scoped_release>()` should work in PyPy as well,
+#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON)
+    // `py::call_guard<py::gil_scoped_release>()` should work in PyPy/GraalPy as well,
     // but it's unclear how to test it without `PyGILState_GetThisThreadState`.
     auto report_gil_status = []() {
         auto is_gil_held = false;

tests/test_call_policies.py

@@ -8,6 +8,7 @@
 
 
 @pytest.mark.xfail("env.PYPY", reason="sometimes comes out 1 off on PyPy", strict=False)
+@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC")
 def test_keep_alive_argument(capture):
     n_inst = ConstructorStats.detail_reg_inst()
     with capture:
@@ -60,6 +61,7 @@ def test_keep_alive_argument(capture):
     assert str(excinfo.value) == "Could not activate keep_alive!"
 
 
+@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC")
 def test_keep_alive_return_value(capture):
     n_inst = ConstructorStats.detail_reg_inst()
     with capture:
@@ -118,6 +120,7 @@ def test_keep_alive_return_value(capture):
 
 # https://foss.heptapod.net/pypy/pypy/-/issues/2447
 @pytest.mark.xfail("env.PYPY", reason="_PyObject_GetDictPtr is unimplemented")
+@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC")
 def test_alive_gc(capture):
     n_inst = ConstructorStats.detail_reg_inst()
     p = m.ParentGC()
@@ -137,6 +140,7 @@ def test_alive_gc(capture):
     )
 
 
+@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC")
 def test_alive_gc_derived(capture):
     class Derived(m.Parent):
         pass
@@ -159,6 +163,7 @@ class Derived(m.Parent):
     )
 
 
+@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC")
 def test_alive_gc_multi_derived(capture):
     class Derived(m.Parent, m.Child):
         def __init__(self):

tests/test_callbacks.cpp

@@ -270,7 +270,7 @@ TEST_SUBMODULE(callbacks, m) {
     m.add_object("custom_function", PyCFunction_New(custom_def, rec_capsule.ptr()));
 
     // This test requires a new ABI version to pass
-#if PYBIND11_INTERNALS_VERSION > 4
+#if PYBIND11_INTERNALS_VERSION > 4 && !defined(GRAALVM_PYTHON)
     // rec_capsule with nullptr name
     py::capsule rec_capsule2(std::malloc(1), [](void *data) { std::free(data); });
     m.add_object("custom_function2", PyCFunction_New(custom_def, rec_capsule2.ptr()));

tests/test_callbacks.py

@@ -90,6 +90,7 @@ def f(*args, **kwargs):
     )
 
 
+@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC")
 def test_lambda_closure_cleanup():
     m.test_lambda_closure_cleanup()
     cstats = m.payload_cstats()
@@ -98,6 +99,7 @@ def test_lambda_closure_cleanup():
     assert cstats.move_constructions >= 1
 
 
+@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC")
 def test_cpp_callable_cleanup():
     alive_counts = m.test_cpp_callable_cleanup()
     assert alive_counts == [0, 1, 2, 1, 2, 1, 0]

tests/test_class.py

@@ -361,7 +361,7 @@ def test_brace_initialization():
     assert b.vec == [123, 456]
 
 
-@pytest.mark.xfail("env.PYPY")
+@pytest.mark.xfail("env.PYPY or env.GRAALPY")
 def test_class_refcount():
     """Instances must correctly increase/decrease the reference count of their types (#1029)"""
     from sys import getrefcount

tests/test_cmake_build/CMakeLists.txt

@@ -55,8 +55,10 @@ possibly_uninitialized(PYTHON_MODULE_EXTENSION Python_INTERPRETER_ID)
 
 pybind11_add_build_test(subdirectory_function)
 pybind11_add_build_test(subdirectory_target)
-if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy")
-  message(STATUS "Skipping embed test on PyPy")
+if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy"
+   OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy"
+   OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy")
+  message(STATUS "Skipping embed test on PyPy or GraalPy")
 else()
   pybind11_add_build_test(subdirectory_embed)
 endif()
@@ -66,10 +68,14 @@ if(PYBIND11_INSTALL)
     mock_install ${CMAKE_COMMAND} "-DCMAKE_INSTALL_PREFIX=${pybind11_BINARY_DIR}/mock_install" -P
                  "${pybind11_BINARY_DIR}/cmake_install.cmake")
 
-  pybind11_add_build_test(installed_function INSTALL)
+  if(NOT "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy")
+    pybind11_add_build_test(installed_function INSTALL)
+  endif()
   pybind11_add_build_test(installed_target INSTALL)
-  if(NOT ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy"
-         ))
+  if(NOT
+     ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy"
+      OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy"
+      OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy"))
     pybind11_add_build_test(installed_embed INSTALL)
   endif()
 endif()

tests/test_cpp_conduit.py

@@ -7,6 +7,7 @@
 import home_planet_very_lonely_traveler
 import pytest
 
+import env
 from pybind11_tests import cpp_conduit as home_planet
 
 
@@ -27,7 +28,10 @@ def test_call_cpp_conduit_success():
         home_planet.cpp_type_info_capsule_Traveler,
         b"raw_pointer_ephemeral",
     )
-    assert cap.__class__.__name__ == "PyCapsule"
+    assert cap.__class__.__name__ == "PyCapsule" or (
+        # Note: this will become unnecessary in the next GraalPy release
+        env.GRAALPY and cap.__class__.__name__ == "capsule"
+    )
 
 
 def test_call_cpp_conduit_platform_abi_id_mismatch():

tests/test_custom_type_setup.py

@@ -34,15 +34,15 @@ def add_ref(obj):
 
 
 # PyPy does not seem to reliably garbage collect.
-@pytest.mark.skipif("env.PYPY")
+@pytest.mark.skipif("env.PYPY or env.GRAALPY")
 def test_self_cycle(gc_tester):
     obj = m.OwnsPythonObjects()
     obj.value = obj
     gc_tester(obj)
 
 
 # PyPy does not seem to reliably garbage collect.
-@pytest.mark.skipif("env.PYPY")
+@pytest.mark.skipif("env.PYPY or env.GRAALPY")
 def test_indirect_cycle(gc_tester):
     obj = m.OwnsPythonObjects()
     obj_list = [obj]

tests/test_eigen_matrix.py

@@ -2,6 +2,7 @@
 
 import pytest
 
+import env  # noqa: F401
 from pybind11_tests import ConstructorStats
 
 np = pytest.importorskip("numpy")
@@ -409,6 +410,7 @@ def assert_keeps_alive(cl, method, *args):
     assert cstats.alive() == start_with
 
 
+@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC")
 def test_eigen_keepalive():
     a = m.ReturnTester()
     cstats = ConstructorStats.get(m.ReturnTester)