From 2ca04bb1b3d99a67e29aed52b05717f90171b1eb Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 28 Apr 2023 17:23:23 +0200 Subject: [PATCH 1/3] gh-103968: Deprecate creating heap types whose metaclass has custom tp_new. (That's a mouthful of an edge case!) --- Doc/c-api/type.rst | 26 ++++++++++++-- Doc/whatsnew/3.12.rst | 20 +++++++++++ Lib/test/test_capi/test_misc.py | 14 ++++++++ ...-04-28-18-04-38.gh-issue-103968.EnVvOx.rst | 4 +++ Modules/_testcapi/heaptype.c | 16 ++++++++- Objects/typeobject.c | 36 ++++++++++++++----- 6 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 69b15296993301..9fd40e1008c4a8 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -256,8 +256,13 @@ The following functions and structs are used to create The metaclass *metaclass* is used to construct the resulting type object. When *metaclass* is ``NULL``, the metaclass is derived from *bases* (or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below). - Note that metaclasses that override - :c:member:`~PyTypeObject.tp_new` are not supported. + + Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not + supported. + (For backwards compatibility, other ``PyType_From*`` functions allow + such metaclasses. They ignore ``tp_new``, which may result in incomplete + initialization. This is deprecated and in Python 3.14+ such metaclasses will + not be supported.) The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. @@ -305,6 +310,11 @@ The following functions and structs are used to create The function now finds and uses a metaclass corresponding to the provided base classes. Previously, only :class:`type` instances were returned. + The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. + which may result in incomplete initialization. + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it + will be no longer allowed. .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -317,6 +327,12 @@ The following functions and structs are used to create The function now finds and uses a metaclass corresponding to the provided base classes. Previously, only :class:`type` instances were returned. + The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. + which may result in incomplete initialization. + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it + will be no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -327,6 +343,12 @@ The following functions and structs are used to create base classes provided in *Py_tp_base[s]* slots. Previously, only :class:`type` instances were returned. + The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. + which may result in incomplete initialization. + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it + will be no longer allowed. + .. c:type:: PyType_Spec Structure defining a type's behavior. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 291500532493c6..76da28efa64919 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1288,6 +1288,21 @@ Porting to Python 3.12 available on debug builds. If you happen to be using it then you'll need to start using ``_Py_GetGlobalRefTotal()``. +* The following functions now select an appropriate metaclass for the newly + created type: + + * :c:func:`PyType_FromSpec` + * :c:func:`PyType_FromSpecWithBases` + * :c:func:`PyType_FromModuleAndSpec` + + Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` + is deprecated, and in Python 3.14+ it will be no longer allowed. + Note that these functions ignore ``tp_new`` of the metaclass, possibly + allowing incomplete initialization. + + Note that :c:func:`PyType_FromMetaclass` (added in Python 3.12) + already disallows creating classes whose metaclass overrides ``tp_new``. + Deprecated ---------- @@ -1364,6 +1379,11 @@ Deprecated * ``_PyErr_ChainExceptions`` is deprecated. Use ``_PyErr_ChainExceptions1`` instead. (Contributed by Irit Katriel in :gh:`102192`.) +* Using :c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases` + or :c:func:`PyType_FromModuleAndSpec` to create a class whose metaclass + overrides :c:member:`~PyTypeObject.tp_new`. + Call the metaclass instead. + Removed ------- diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 9470cf12a7d1c4..b183736ca6f9e7 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -681,6 +681,20 @@ def test_heaptype_with_custom_metaclass(self): with self.assertRaisesRegex(TypeError, msg): t = _testcapi.pytype_fromspec_meta(_testcapi.HeapCTypeMetaclassCustomNew) + def test_heaptype_with_custom_metaclass_deprecation(self): + # gh-103968: a metaclass with custom tp_new is deprecated, but still + # allowed for functions that existed in 3.11 + # (PyType_FromSpecWithBases is used here). + class Base(metaclass=_testcapi.HeapCTypeMetaclassCustomNew): + pass + + with warnings_helper.check_warnings( + ('.*custom tp_new.*in Python 3.14.*', DeprecationWarning), + ): + sub = _testcapi.make_type_with_base(Base) + self.assertTrue(issubclass(sub, Base)) + self.assertIsInstance(sub, _testcapi.HeapCTypeMetaclassCustomNew) + def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): with self.assertRaises(TypeError): diff --git a/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst b/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst new file mode 100644 index 00000000000000..5e4270f82afd84 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst @@ -0,0 +1,4 @@ +:c:func:`PyType_FromSpec` and its variants now allow creating classes whose +metaclass overrides :c:member:`~PyTypeObject.tp_new`. The ``tp_new`` is +ignored. This behavior is deprecated and will be disallowed in 3.14+. The +new :c:func:`PyType_FromMetaclass` already disallows it. diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 209cc182c0698d..6384fbc485fbdf 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -22,7 +22,7 @@ static PyObject *pytype_fromspec_meta(PyObject* self, PyObject *meta) "_testcapi.HeapCTypeViaMetaclass", sizeof(PyObject), 0, - Py_TPFLAGS_DEFAULT, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, HeapCTypeViaMetaclass_slots }; @@ -385,6 +385,19 @@ make_immutable_type_with_base(PyObject *self, PyObject *base) return PyType_FromSpecWithBases(&ImmutableSubclass_spec, base); } +static PyObject * +make_type_with_base(PyObject *self, PyObject *base) +{ + assert(PyType_Check(base)); + PyType_Spec ImmutableSubclass_spec = { + .name = "_testcapi.Subclass", + .basicsize = (int)((PyTypeObject*)base)->tp_basicsize, + .slots = empty_type_slots, + .flags = Py_TPFLAGS_DEFAULT, + }; + return PyType_FromSpecWithBases(&ImmutableSubclass_spec, base); +} + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, @@ -397,6 +410,7 @@ static PyMethodDef TestMethods[] = { test_from_spec_invalid_metatype_inheritance, METH_NOARGS}, {"make_immutable_type_with_base", make_immutable_type_with_base, METH_O}, + {"make_type_with_base", make_type_with_base, METH_O}, {NULL}, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 38b99315457a58..b1094a3879f146 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3634,8 +3634,9 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type) } PyObject * -PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in) +_PyType_FromMetaclass_impl( + PyTypeObject *metaclass, PyObject *module, + PyType_Spec *spec, PyObject *bases_in, int _allow_tp_new) { /* Invariant: A non-NULL value in one of these means this function holds * a strong reference or owns allocated memory. @@ -3810,9 +3811,21 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, goto finally; } if (metaclass->tp_new != PyType_Type.tp_new) { - PyErr_SetString(PyExc_TypeError, - "Metaclasses with custom tp_new are not supported."); - goto finally; + if (_allow_tp_new) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Using PyType_Spec with metaclasses that have custom " + "tp_new is deprecated and will no longer be allowed in " + "Python 3.14.") < 0) { + goto finally; + } + } + else { + PyErr_SetString( + PyExc_TypeError, + "Metaclasses with custom tp_new are not supported."); + goto finally; + } } /* Calculate best base, and check that all bases are type objects */ @@ -3998,22 +4011,29 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, return (PyObject*)res; } +PyObject * +PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, + PyType_Spec *spec, PyObject *bases_in) +{ + return _PyType_FromMetaclass_impl(metaclass, module, spec, bases_in, 0); +} + PyObject * PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { - return PyType_FromMetaclass(NULL, module, spec, bases); + return _PyType_FromMetaclass_impl(NULL, module, spec, bases, 1); } PyObject * PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { - return PyType_FromMetaclass(NULL, NULL, spec, bases); + return _PyType_FromMetaclass_impl(NULL, NULL, spec, bases, 1); } PyObject * PyType_FromSpec(PyType_Spec *spec) { - return PyType_FromMetaclass(NULL, NULL, spec, NULL); + return _PyType_FromMetaclass_impl(NULL, NULL, spec, NULL, 1); } PyObject * From 492198728a79737babd8e29be943fe4e658ea2fc Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 2 May 2023 17:28:47 +0200 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Barney Gale --- Doc/whatsnew/3.12.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 76da28efa64919..42ad8fe047b2e3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1296,7 +1296,7 @@ Porting to Python 3.12 * :c:func:`PyType_FromModuleAndSpec` Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` - is deprecated, and in Python 3.14+ it will be no longer allowed. + is deprecated, and in Python 3.14+ it will be disallowed. Note that these functions ignore ``tp_new`` of the metaclass, possibly allowing incomplete initialization. @@ -1381,7 +1381,7 @@ Deprecated * Using :c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases` or :c:func:`PyType_FromModuleAndSpec` to create a class whose metaclass - overrides :c:member:`~PyTypeObject.tp_new`. + overrides :c:member:`~PyTypeObject.tp_new` is deprecated. Call the metaclass instead. Removed From ecaf7d9e4f6f746df0bc717a818cb1cd819cf5ca Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 2 May 2023 17:30:02 +0200 Subject: [PATCH 3/3] Add `static` to _PyType_FromMetaclass_impl --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index edcc72cb3bba7a..8ab03b3a6a2803 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3830,7 +3830,7 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type) return 1; } -PyObject * +static PyObject * _PyType_FromMetaclass_impl( PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases_in, int _allow_tp_new)