From 1d3392517698170e270eb7d847b6a8c28bfaca0f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Apr 2024 19:13:44 -0600 Subject: [PATCH 01/22] gh-110693: Use a Larger Queue for Per-Interpreter Pending Calls (gh-118302) This is an improvement over the status quo, reducing the likelihood of completely filling the pending calls queue. However, the problem won't go away completely unless we move to an unbounded linked list or add a mechanism for waiting until the queue isn't full. --- Include/internal/pycore_ceval_state.h | 6 ++++-- Lib/test/test_capi/test_misc.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 1831f58899b745..11f2a100bf531e 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -20,7 +20,7 @@ struct _pending_call { int flags; }; -#define PENDINGCALLSARRAYSIZE 32 +#define PENDINGCALLSARRAYSIZE 300 #define MAXPENDINGCALLS PENDINGCALLSARRAYSIZE /* For interpreter-level pending calls, we want to avoid spending too @@ -31,7 +31,9 @@ struct _pending_call { # define MAXPENDINGCALLSLOOP MAXPENDINGCALLS #endif -#define MAXPENDINGCALLS_MAIN PENDINGCALLSARRAYSIZE +/* We keep the number small to preserve as much compatibility + as possible with earlier versions. */ +#define MAXPENDINGCALLS_MAIN 32 /* For the main thread, we want to make sure all pending calls are run at once, for the sake of prompt signal handling. This is unlikely to cause any problems since there should be very few diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 49d1056f050467..020e8493e57c0c 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1570,9 +1570,9 @@ def test_max_pending(self): self.assertEqual(added, maxpending) with self.subTest('not main-only'): - # Per-interpreter pending calls has the same low limit + # Per-interpreter pending calls has a much higher limit # on how many may be pending at a time. - maxpending = 32 + maxpending = 300 l = [] added = self.pendingcalls_submit(l, 1, main=False) From 8397d8d3002d3d817d4fbb8f852690be8ed0d6c7 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Sat, 27 Apr 2024 10:06:08 +0800 Subject: [PATCH 02/22] Correct spelling error in recent NEWS entry (#118308) --- .../next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst b/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst index 8eb68ebe99ba15..daa5fe7c0f2917 100644 --- a/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst +++ b/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst @@ -6,4 +6,4 @@ class was dynamically created, the class held strong references to other objects which took up a significant amount of memory, and the cache contained the sole strong reference to the class. The fix for the regression leads to a slowdown in :func:`getattr_static`, but the function should still -be signficantly faster than it was in Python 3.11. Patch by Alex Waygood. +be significantly faster than it was in Python 3.11. Patch by Alex Waygood. From c57326f48729f5cd7ddf7e2b38c4fd06d0962a41 Mon Sep 17 00:00:00 2001 From: Wulian233 <71213467+Wulian233@users.noreply.github.com> Date: Sat, 27 Apr 2024 17:25:32 +0800 Subject: [PATCH 03/22] Correct typo in iOS README (#118341) --- iOS/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iOS/README.rst b/iOS/README.rst index df429b64cec77f..a0ffa5737aae43 100644 --- a/iOS/README.rst +++ b/iOS/README.rst @@ -50,7 +50,7 @@ iOS specific arguments to configure Unless you know what you're doing, changing the name of the Python framework on iOS is not advised. If you use this option, you won't be able - to run the ``make testios`` target without making signficant manual + to run the ``make testios`` target without making significant manual alterations, and you won't be able to use any binary packages unless you compile them yourself using your own framework name. From 51aefc5bf907ddffaaf083ded0de773adcdf08c8 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 27 Apr 2024 13:36:06 +0300 Subject: [PATCH 04/22] gh-118323: Document `&&` grammar syntax (#118324) --- Grammar/python.gram | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Grammar/python.gram b/Grammar/python.gram index 3943d7fec5db03..05d7837e3aa6db 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -78,6 +78,9 @@ _PyPegen_parse(Parser *p) # Fail if e can be parsed, without consuming any input. # ~ # Commit to the current alternative, even if it fails to parse. +# &&e +# Eager parse e. The parser will not backtrack and will immediately +# fail with SyntaxError if e cannot be parsed. # # STARTING RULES From 2326d6c868e300a814179d77fc308fec8365cb8c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 28 Apr 2024 06:21:28 -0700 Subject: [PATCH 05/22] gh-109118: Make comprehensions work within annotation scopes, but without inlining (#118160) Co-authored-by: Carl Meyer --- Doc/whatsnew/3.13.rst | 4 +- Lib/test/test_type_params.py | 48 +++++++++++-------- ...-04-25-21-18-19.gh-issue-118160.GH5SMc.rst | 3 ++ Python/symtable.c | 18 ++----- 4 files changed, 39 insertions(+), 34 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-25-21-18-19.gh-issue-118160.GH5SMc.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 083a70ce2405e3..98349a5984bb7e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -276,7 +276,9 @@ Other Language Changes (Contributed by Pedro Sousa Lacerda in :gh:`66449`.) * :ref:`annotation scope ` within class scopes can now - contain lambdas. (Contributed by Jelle Zijlstra in :gh:`109118`.) + contain lambdas and comprehensions. Comprehensions that are located within + class scopes are not inlined into their parent scope. (Contributed by + Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) New Modules diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index fbb80d9aac9942..4b86395ee74f75 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -436,9 +436,11 @@ class C[T]: class Inner[U](make_base(T for _ in (1,)), make_base(T)): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(list(base1.__arg__), [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") def test_listcomp_in_nested_class(self): code = """ @@ -464,9 +466,11 @@ class C[T]: class Inner[U](make_base([T for _ in (1,)]), make_base(T)): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(base1.__arg__, [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") def test_gen_exp_in_generic_method(self): code = """ @@ -475,27 +479,33 @@ class C[T]: def meth[U](x: (T for _ in (1,)), y: T): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + meth = ns["C"].meth + self.assertEqual(list(meth.__annotations__["x"]), [ns["C"].__type_params__[0]]) + self.assertEqual(meth.__annotations__["y"], "class") def test_nested_scope_in_generic_alias(self): code = """ - class C[T]: + T = "global" + class C: T = "class" {} """ - error_cases = [ - "type Alias3[T] = (T for _ in (1,))", - "type Alias4 = (T for _ in (1,))", - "type Alias5[T] = [T for _ in (1,)]", - "type Alias6 = [T for _ in (1,)]", + cases = [ + "type Alias[T] = (T for _ in (1,))", + "type Alias = (T for _ in (1,))", + "type Alias[T] = [T for _ in (1,)]", + "type Alias = [T for _ in (1,)]", ] - for case in error_cases: + for case in cases: with self.subTest(case=case): - with self.assertRaisesRegex(SyntaxError, - r"Cannot use [a-z]+ in annotation scope within class scope"): - run_code(code.format(case)) + ns = run_code(code.format(case)) + alias = ns["C"].Alias + value = list(alias.__value__)[0] + if alias.__type_params__: + self.assertIs(value, alias.__type_params__[0]) + else: + self.assertEqual(value, "global") def test_lambda_in_alias_in_class(self): code = """ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-25-21-18-19.gh-issue-118160.GH5SMc.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-25-21-18-19.gh-issue-118160.GH5SMc.rst new file mode 100644 index 00000000000000..c4e798df5de702 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-25-21-18-19.gh-issue-118160.GH5SMc.rst @@ -0,0 +1,3 @@ +:ref:`Annotation scopes ` within classes can now contain +comprehensions. However, such comprehensions are not inlined into their +parent scope at runtime. Patch by Jelle Zijlstra. diff --git a/Python/symtable.c b/Python/symtable.c index 483ef1c3c46542..eecd159b2c3f17 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1154,10 +1154,12 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } } - // we inline all non-generator-expression comprehensions + // we inline all non-generator-expression comprehensions, + // except those in annotation scopes that are nested in classes int inline_comp = entry->ste_comprehension && - !entry->ste_generator; + !entry->ste_generator && + !ste->ste_can_see_class_scope; if (!analyze_child_block(entry, newbound, newfree, newglobal, type_params, new_class_entry, &child_free)) @@ -2589,18 +2591,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, identifier scope_name, asdl_comprehension_seq *generators, expr_ty elt, expr_ty value) { - if (st->st_cur->ste_can_see_class_scope) { - // gh-109118 - PyErr_Format(PyExc_SyntaxError, - "Cannot use comprehension in annotation scope within class scope"); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); - VISIT_QUIT(st, 0); - } - int is_generator = (e->kind == GeneratorExp_kind); comprehension_ty outermost = ((comprehension_ty) asdl_seq_GET(generators, 0)); From c618d53a3a09c4b145c54fd9da4ca367be70f1e4 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Mon, 29 Apr 2024 01:00:48 +0800 Subject: [PATCH 06/22] Fix typo in Tools/wasm/README.md(#118358) --- Tools/wasm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index eca113d3bfabad..bc3e4ba8bd5b76 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -275,7 +275,7 @@ Node builds use ``NODERAWFS``. ### Hosting Python WASM builds The simple REPL terminal uses SharedArrayBuffer. For security reasons -browsers only provide the feature in secure environents with cross-origin +browsers only provide the feature in secure environments with cross-origin isolation. The webserver must send cross-origin headers and correct MIME types for the JavaScript and WASM files. Otherwise the terminal will fail to load with an error message like ``Browsers disable shared array buffer``. From e0ab6424369bcd276b0dc4f537fde360bdca2f05 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 28 Apr 2024 20:31:22 +0300 Subject: [PATCH 07/22] gh-101100: Fix Sphinx warnings in `whatsnew/3.9.rst` (#118364) --- Doc/whatsnew/3.7.rst | 6 ++--- Doc/whatsnew/3.9.rst | 48 ++++++++++++++++++++-------------------- Misc/NEWS.d/3.10.0a1.rst | 4 ++-- Misc/NEWS.d/3.11.0a4.rst | 2 +- Misc/NEWS.d/3.9.0a1.rst | 2 +- Misc/NEWS.d/3.9.0a2.rst | 2 +- Misc/NEWS.d/3.9.0a6.rst | 10 ++++----- 7 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index ad7c8b5320180e..69d043bcf7efd5 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -669,8 +669,8 @@ include: * The new :func:`asyncio.current_task` function returns the currently running :class:`~asyncio.Task` instance, and the new :func:`asyncio.all_tasks` function returns a set of all existing ``Task`` instances in a given loop. - The :meth:`Task.current_task() ` and - :meth:`Task.all_tasks() ` methods have been deprecated. + The :meth:`!Task.current_task` and + :meth:`!Task.all_tasks` methods have been deprecated. (Contributed by Andrew Svetlov in :issue:`32250`.) * The new *provisional* :class:`~asyncio.BufferedProtocol` class allows @@ -1969,7 +1969,7 @@ asynchronous context manager must be used in order to acquire and release the synchronization resource. (Contributed by Andrew Svetlov in :issue:`32253`.) -The :meth:`asyncio.Task.current_task` and :meth:`asyncio.Task.all_tasks` +The :meth:`!asyncio.Task.current_task` and :meth:`!asyncio.Task.all_tasks` methods have been deprecated. (Contributed by Andrew Svetlov in :issue:`32250`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index de248bc3584d9a..e29d37ca120b76 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -81,13 +81,13 @@ Interpreter improvements: * a number of Python builtins (range, tuple, set, frozenset, list, dict) are now sped up using :pep:`590` vectorcall; * garbage collection does not block on resurrected objects; -* a number of Python modules (:mod:`_abc`, :mod:`!audioop`, :mod:`_bz2`, - :mod:`_codecs`, :mod:`_contextvars`, :mod:`!_crypt`, :mod:`_functools`, - :mod:`_json`, :mod:`_locale`, :mod:`math`, :mod:`operator`, :mod:`resource`, - :mod:`time`, :mod:`_weakref`) now use multiphase initialization as defined +* a number of Python modules (:mod:`!_abc`, :mod:`!audioop`, :mod:`!_bz2`, + :mod:`!_codecs`, :mod:`!_contextvars`, :mod:`!_crypt`, :mod:`!_functools`, + :mod:`!_json`, :mod:`!_locale`, :mod:`math`, :mod:`operator`, :mod:`resource`, + :mod:`time`, :mod:`!_weakref`) now use multiphase initialization as defined by PEP 489; * a number of standard library modules (:mod:`!audioop`, :mod:`ast`, :mod:`grp`, - :mod:`_hashlib`, :mod:`pwd`, :mod:`_posixsubprocess`, :mod:`random`, + :mod:`!_hashlib`, :mod:`pwd`, :mod:`!_posixsubprocess`, :mod:`random`, :mod:`select`, :mod:`struct`, :mod:`termios`, :mod:`zlib`) are now using the stable ABI defined by PEP 384. @@ -203,7 +203,7 @@ The :mod:`ast` module uses the new parser and produces the same AST as the old parser. In Python 3.10, the old parser will be deleted and so will all -functionality that depends on it (primarily the :mod:`parser` module, +functionality that depends on it (primarily the :mod:`!parser` module, which has long been deprecated). In Python 3.9 *only*, you can switch back to the LL(1) parser using a command line switch (``-X oldparser``) or an environment variable (``PYTHONOLDPARSER=1``). @@ -366,7 +366,7 @@ wait until the cancellation is complete also in the case when *timeout* is <= 0, like it does with positive timeouts. (Contributed by Elvis Pranskevichus in :issue:`32751`.) -:mod:`asyncio` now raises :exc:`TyperError` when calling incompatible +:mod:`asyncio` now raises :exc:`TypeError` when calling incompatible methods with an :class:`ssl.SSLSocket` socket. (Contributed by Ido Michael in :issue:`37404`.) @@ -589,7 +589,7 @@ a non-blocking socket. (Contributed by Donghee Na in :issue:`39259`.) os -- -Added :const:`~os.CLD_KILLED` and :const:`~os.CLD_STOPPED` for :attr:`si_code`. +Added :const:`~os.CLD_KILLED` and :const:`~os.CLD_STOPPED` for :attr:`!si_code`. (Contributed by Donghee Na in :issue:`38493`.) Exposed the Linux-specific :func:`os.pidfd_open` (:issue:`38692`) and @@ -861,7 +861,7 @@ Deprecated Python versions it will raise a :exc:`TypeError` for all floats. (Contributed by Serhiy Storchaka in :issue:`37315`.) -* The :mod:`parser` and :mod:`symbol` modules are deprecated and will be +* The :mod:`!parser` and :mod:`!symbol` modules are deprecated and will be removed in future versions of Python. For the majority of use cases, users can leverage the Abstract Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` module. @@ -889,7 +889,7 @@ Deprecated it for writing and silencing a warning. (Contributed by Serhiy Storchaka in :issue:`28286`.) -* Deprecated the ``split()`` method of :class:`_tkinter.TkappType` in +* Deprecated the ``split()`` method of :class:`!_tkinter.TkappType` in favour of the ``splitlist()`` method which has more consistent and predicable behavior. (Contributed by Serhiy Storchaka in :issue:`38371`.) @@ -898,11 +898,11 @@ Deprecated deprecated and will be removed in version 3.11. (Contributed by Yury Selivanov and Kyle Stanley in :issue:`34790`.) -* binhex4 and hexbin4 standards are now deprecated. The :mod:`binhex` module +* binhex4 and hexbin4 standards are now deprecated. The :mod:`!binhex` module and the following :mod:`binascii` functions are now deprecated: - * :func:`~binascii.b2a_hqx`, :func:`~binascii.a2b_hqx` - * :func:`~binascii.rlecode_hqx`, :func:`~binascii.rledecode_hqx` + * :func:`!b2a_hqx`, :func:`!a2b_hqx` + * :func:`!rlecode_hqx`, :func:`!rledecode_hqx` (Contributed by Victor Stinner in :issue:`39353`.) @@ -950,7 +950,7 @@ Deprecated Removed ======= -* The erroneous version at :data:`unittest.mock.__version__` has been removed. +* The erroneous version at :data:`!unittest.mock.__version__` has been removed. * :class:`!nntplib.NNTP`: ``xpath()`` and ``xgtitle()`` methods have been removed. These methods are deprecated since Python 3.3. Generally, these extensions @@ -987,7 +987,7 @@ Removed removed. They were deprecated since Python 3.7. (Contributed by Victor Stinner in :issue:`37320`.) -* The :meth:`~threading.Thread.isAlive()` method of :class:`threading.Thread` +* The :meth:`!isAlive()` method of :class:`threading.Thread` has been removed. It was deprecated since Python 3.8. Use :meth:`~threading.Thread.is_alive()` instead. (Contributed by Donghee Na in :issue:`37804`.) @@ -1035,7 +1035,7 @@ Removed ``asyncio.Condition`` and ``asyncio.Semaphore``. (Contributed by Andrew Svetlov in :issue:`34793`.) -* The :func:`sys.getcounts` function, the ``-X showalloccount`` command line +* The :func:`!sys.getcounts` function, the ``-X showalloccount`` command line option and the ``show_alloc_count`` field of the C structure :c:type:`PyConfig` have been removed. They required a special Python build by defining ``COUNT_ALLOCS`` macro. @@ -1046,11 +1046,11 @@ Removed the ``__annotations__`` attribute instead. (Contributed by Serhiy Storchaka in :issue:`40182`.) -* The :meth:`symtable.SymbolTable.has_exec` method has been removed. It was +* The :meth:`!symtable.SymbolTable.has_exec` method has been removed. It was deprecated since 2006, and only returning ``False`` when it's called. (Contributed by Batuhan Taskaya in :issue:`40208`) -* The :meth:`asyncio.Task.current_task` and :meth:`asyncio.Task.all_tasks` +* The :meth:`!asyncio.Task.current_task` and :meth:`!asyncio.Task.all_tasks` have been removed. They were deprecated since Python 3.7 and you can use :func:`asyncio.current_task` and :func:`asyncio.all_tasks` instead. (Contributed by Rémi Lapeyre in :issue:`40967`) @@ -1230,7 +1230,7 @@ Build Changes * The ``COUNT_ALLOCS`` special build macro has been removed. (Contributed by Victor Stinner in :issue:`39489`.) -* On non-Windows platforms, the :c:func:`setenv` and :c:func:`unsetenv` +* On non-Windows platforms, the :c:func:`!setenv` and :c:func:`!unsetenv` functions are now required to build Python. (Contributed by Victor Stinner in :issue:`39395`.) @@ -1319,7 +1319,7 @@ New Features the garbage collector respectively. (Contributed by Pablo Galindo Salgado in :issue:`40241`.) -* Added :c:func:`_PyObject_FunctionStr` to get a user-friendly string +* Added :c:func:`!_PyObject_FunctionStr` to get a user-friendly string representation of a function-like object. (Patch by Jeroen Demeyer in :issue:`37645`.) @@ -1361,7 +1361,7 @@ Porting to Python 3.9 and refers to a constant string. (Contributed by Serhiy Storchaka in :issue:`38650`.) -* The :c:type:`PyGC_Head` structure is now opaque. It is only defined in the +* The :c:type:`!PyGC_Head` structure is now opaque. It is only defined in the internal C API (``pycore_gc.h``). (Contributed by Victor Stinner in :issue:`40241`.) @@ -1384,12 +1384,12 @@ Porting to Python 3.9 * :c:func:`PyObject_IS_GC` macro was converted to a function. - * The :c:func:`PyObject_NEW` macro becomes an alias to the - :c:macro:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro + * The :c:func:`!PyObject_NEW` macro becomes an alias to the + :c:macro:`PyObject_New` macro, and the :c:func:`!PyObject_NEW_VAR` macro becomes an alias to the :c:macro:`PyObject_NewVar` macro. They no longer access directly the :c:member:`PyTypeObject.tp_basicsize` member. - * :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: + * :c:func:`!PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: the macro accessed directly the :c:member:`PyTypeObject.tp_weaklistoffset` member. diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index 2e32ca9f3b26bb..9a729a45b160eb 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -1861,8 +1861,8 @@ bundled versions of ``pip`` and ``setuptools``. Patch by Krzysztof Konopko. .. nonce: _dx3OO .. section: Library -Removed :meth:`asyncio.Task.current_task` and -:meth:`asyncio.Task.all_tasks`. Patch contributed by Rémi Lapeyre. +Removed :meth:`!asyncio.Task.current_task` and +:meth:`!asyncio.Task.all_tasks`. Patch contributed by Rémi Lapeyre. .. diff --git a/Misc/NEWS.d/3.11.0a4.rst b/Misc/NEWS.d/3.11.0a4.rst index 78b682f7a22cc6..a5ce7620016cc7 100644 --- a/Misc/NEWS.d/3.11.0a4.rst +++ b/Misc/NEWS.d/3.11.0a4.rst @@ -7,7 +7,7 @@ :c:func:`Py_EndInterpreter` now explicitly untracks all objects currently tracked by the GC. Previously, if an object was used later by another interpreter, calling :c:func:`PyObject_GC_UnTrack` on the object crashed if -the previous or the next object of the :c:type:`PyGC_Head` structure became +the previous or the next object of the :c:type:`!PyGC_Head` structure became a dangling pointer. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 8f38f04eb41798..39d760cdd4fddf 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -5616,7 +5616,7 @@ heap type .. nonce: 4DcUaI .. section: C API -Add :c:func:`_PyObject_FunctionStr` to get a user-friendly string +Add :c:func:`!_PyObject_FunctionStr` to get a user-friendly string representation of a function-like object. Patch by Jeroen Demeyer. .. diff --git a/Misc/NEWS.d/3.9.0a2.rst b/Misc/NEWS.d/3.9.0a2.rst index 7d878cfe227552..39b1c308312aa4 100644 --- a/Misc/NEWS.d/3.9.0a2.rst +++ b/Misc/NEWS.d/3.9.0a2.rst @@ -844,7 +844,7 @@ test.regrtest now can receive a list of test patterns to ignore (using the .. nonce: cNsA7S .. section: Build -:mod:`asyncio` now raises :exc:`TyperError` when calling incompatible +:mod:`asyncio` now raises :exc:`TypeError` when calling incompatible methods with an :class:`ssl.SSLSocket` socket. Patch by Ido Michael. .. diff --git a/Misc/NEWS.d/3.9.0a6.rst b/Misc/NEWS.d/3.9.0a6.rst index 26a6fb98efdc36..466ff624fcbf81 100644 --- a/Misc/NEWS.d/3.9.0a6.rst +++ b/Misc/NEWS.d/3.9.0a6.rst @@ -564,7 +564,7 @@ Implement traverse and clear slots in _abc._abc_data type. .. nonce: 3rO_q7 .. section: Library -Remove deprecated :meth:`symtable.SymbolTable.has_exec`. +Remove deprecated :meth:`!symtable.SymbolTable.has_exec`. .. @@ -1118,7 +1118,7 @@ into an exit code. .. nonce: _FOf7E .. section: C API -Move the :c:type:`PyGC_Head` structure to the internal C API. +Move the :c:type:`!PyGC_Head` structure to the internal C API. .. @@ -1149,8 +1149,8 @@ the garbage collector respectively. Patch by Pablo Galindo. .. nonce: Seuh3D .. section: C API -The :c:func:`PyObject_NEW` macro becomes an alias to the -:c:func:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro +The :c:func:`!PyObject_NEW` macro becomes an alias to the +:c:func:`PyObject_New` macro, and the :c:func:`!PyObject_NEW_VAR` macro becomes an alias to the :c:func:`PyObject_NewVar` macro, to hide implementation details. They no longer access directly the :c:member:`PyTypeObject.tp_basicsize` member. @@ -1174,7 +1174,7 @@ used. .. nonce: 6nFYbY .. section: C API -Convert the :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro to a function to +Convert the :c:func:`!PyObject_GET_WEAKREFS_LISTPTR` macro to a function to hide implementation details: the macro accessed directly to the :c:member:`PyTypeObject.tp_weaklistoffset` member. From 33c6cf3148781993aa1665b1f2231d5442df8f40 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 28 Apr 2024 21:06:45 +0300 Subject: [PATCH 08/22] gh-101100: Fix Sphinx warnings in `library/faulthandler.rst` (#118353) --- Doc/library/faulthandler.rst | 16 +++++++++------- Doc/tools/.nitignore | 1 - 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 96593ee97a139d..c40d5e9aacb83c 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -10,14 +10,15 @@ This module contains functions to dump Python tracebacks explicitly, on a fault, after a timeout, or on a user signal. Call :func:`faulthandler.enable` to -install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`, -:const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also +install fault handlers for the :const:`~signal.SIGSEGV`, +:const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS`, and +:const:`~signal.SIGILL` signals. You can also enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment variable or by using the :option:`-X` ``faulthandler`` command line option. The fault handler is compatible with system fault handlers like Apport or the Windows fault handler. The module uses an alternative stack for signal handlers -if the :c:func:`sigaltstack` function is available. This allows it to dump the +if the :c:func:`!sigaltstack` function is available. This allows it to dump the traceback even on a stack overflow. The fault handler is called on catastrophic cases and therefore can only use @@ -70,8 +71,9 @@ Fault handler state .. function:: enable(file=sys.stderr, all_threads=True) - Enable the fault handler: install handlers for the :const:`SIGSEGV`, - :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` + Enable the fault handler: install handlers for the :const:`~signal.SIGSEGV`, + :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` + and :const:`~signal.SIGILL` signals to dump the Python traceback. If *all_threads* is ``True``, produce tracebacks for every running thread. Otherwise, dump only the current thread. @@ -106,8 +108,8 @@ Dumping the tracebacks after a timeout Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call - :c:func:`_exit` with status=1 after dumping the tracebacks. (Note - :c:func:`_exit` exits the process immediately, which means it doesn't do any + :c:func:`!_exit` with status=1 after dumping the tracebacks. (Note + :c:func:`!_exit` exits the process immediately, which means it doesn't do any cleanup like flushing file buffers.) If the function is called twice, the new call replaces previous parameters and resets the timeout. The timer has a sub-second resolution. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 6f38f36d3311c0..4790136a75cba9 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -28,7 +28,6 @@ Doc/library/email.errors.rst Doc/library/email.parser.rst Doc/library/email.policy.rst Doc/library/exceptions.rst -Doc/library/faulthandler.rst Doc/library/functools.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst From f5b7e397c0a0e180257450843ab622ab8783adf6 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 28 Apr 2024 21:12:25 +0300 Subject: [PATCH 09/22] gh-101100: Fix Sphinx warnings in `whatsnew/3.10.rst` (#118356) --- Doc/whatsnew/3.10.rst | 74 +++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 1a4ecdf1737303..b939ccd17903f2 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -352,7 +352,7 @@ was expecting an indentation, including the location of the statement: AttributeErrors ~~~~~~~~~~~~~~~ -When printing :exc:`AttributeError`, :c:func:`PyErr_Display` will offer +When printing :exc:`AttributeError`, :c:func:`!PyErr_Display` will offer suggestions of similar attribute names in the object that the exception was raised from: @@ -366,14 +366,14 @@ raised from: (Contributed by Pablo Galindo in :issue:`38530`.) .. warning:: - Notice this won't work if :c:func:`PyErr_Display` is not called to display the error + Notice this won't work if :c:func:`!PyErr_Display` is not called to display the error which can happen if some other custom error display function is used. This is a common scenario in some REPLs like IPython. NameErrors ~~~~~~~~~~ -When printing :exc:`NameError` raised by the interpreter, :c:func:`PyErr_Display` +When printing :exc:`NameError` raised by the interpreter, :c:func:`!PyErr_Display` will offer suggestions of similar variable names in the function that the exception was raised from: @@ -388,7 +388,7 @@ was raised from: (Contributed by Pablo Galindo in :issue:`38530`.) .. warning:: - Notice this won't work if :c:func:`PyErr_Display` is not called to display the error, + Notice this won't work if :c:func:`!PyErr_Display` is not called to display the error, which can happen if some other custom error display function is used. This is a common scenario in some REPLs like IPython. @@ -690,7 +690,7 @@ are in :pep:`635`, and a longer tutorial is in :pep:`636`. Optional ``EncodingWarning`` and ``encoding="locale"`` option ------------------------------------------------------------- -The default encoding of :class:`TextIOWrapper` and :func:`open` is +The default encoding of :class:`~io.TextIOWrapper` and :func:`open` is platform and locale dependent. Since UTF-8 is used on most Unix platforms, omitting ``encoding`` option when opening UTF-8 files (e.g. JSON, YAML, TOML, Markdown) is a very common bug. For example:: @@ -785,7 +785,7 @@ especially when forward references or invalid types were involved. Compare:: StrCache = 'Cache[str]' # a type alias LOG_PREFIX = 'LOG[DEBUG]' # a module constant -Now the :mod:`typing` module has a special value :data:`TypeAlias` +Now the :mod:`typing` module has a special value :data:`~typing.TypeAlias` which lets you declare type aliases more explicitly:: StrCache: TypeAlias = 'Cache[str]' # a type alias @@ -798,10 +798,10 @@ See :pep:`613` for more details. PEP 647: User-Defined Type Guards --------------------------------- -:data:`TypeGuard` has been added to the :mod:`typing` module to annotate +:data:`~typing.TypeGuard` has been added to the :mod:`typing` module to annotate type guard functions and improve information provided to static type checkers -during type narrowing. For more information, please see :data:`TypeGuard`\ 's -documentation, and :pep:`647`. +during type narrowing. For more information, please see +:data:`~typing.TypeGuard`\ 's documentation, and :pep:`647`. (Contributed by Ken Jin and Guido van Rossum in :issue:`43766`. PEP written by Eric Traut.) @@ -972,8 +972,8 @@ and objects representing asynchronously released resources. Add asynchronous context manager support to :func:`contextlib.nullcontext`. (Contributed by Tom Gringauz in :issue:`41543`.) -Add :class:`AsyncContextDecorator`, for supporting usage of async context managers -as decorators. +Add :class:`~contextlib.AsyncContextDecorator`, for supporting usage of async +context managers as decorators. curses ------ @@ -1089,8 +1089,8 @@ encodings enum ---- -:class:`Enum` :func:`__repr__` now returns ``enum_name.member_name`` and -:func:`__str__` now returns ``member_name``. Stdlib enums available as +:class:`~enum.Enum` :func:`~object.__repr__` now returns ``enum_name.member_name`` and +:func:`~object.__str__` now returns ``member_name``. Stdlib enums available as module constants have a :func:`repr` of ``module_name.member_name``. (Contributed by Ethan Furman in :issue:`40066`.) @@ -1104,7 +1104,7 @@ Add *encoding* and *errors* parameters in :func:`fileinput.input` and :class:`fileinput.FileInput`. (Contributed by Inada Naoki in :issue:`43712`.) -:func:`fileinput.hook_compressed` now returns :class:`TextIOWrapper` object +:func:`fileinput.hook_compressed` now returns :class:`~io.TextIOWrapper` object when *mode* is "r" and file is compressed, like uncompressed files. (Contributed by Inada Naoki in :issue:`5758`.) @@ -1202,12 +1202,12 @@ Feature parity with ``importlib_metadata`` 4.6 :ref:`importlib.metadata entry points ` now provide a nicer experience for selecting entry points by group and name through a new -:class:`importlib.metadata.EntryPoints` class. See the Compatibility +:ref:`importlib.metadata.EntryPoints ` class. See the Compatibility Note in the docs for more info on the deprecation and usage. -Added :func:`importlib.metadata.packages_distributions` for resolving -top-level Python modules and packages to their -:class:`importlib.metadata.Distribution`. +Added :ref:`importlib.metadata.packages_distributions() ` +for resolving top-level Python modules and packages to their +:ref:`importlib.metadata.Distribution `. inspect ------- @@ -1224,7 +1224,7 @@ best practice for accessing the annotations dict defined on any Python object; for more information on best practices for working with annotations, please see :ref:`annotations-howto`. Relatedly, :func:`inspect.signature`, -:func:`inspect.Signature.from_callable`, and :func:`inspect.Signature.from_function` +:func:`inspect.Signature.from_callable`, and :func:`!inspect.Signature.from_function` now call :func:`inspect.get_annotations` to retrieve annotations. This means :func:`inspect.signature` and :func:`inspect.Signature.from_callable` can also now un-stringize stringized annotations. @@ -1484,9 +1484,9 @@ is a :class:`typing.TypedDict`. Subclasses of ``typing.Protocol`` which only have data variables declared will now raise a ``TypeError`` when checked with ``isinstance`` unless they -are decorated with :func:`runtime_checkable`. Previously, these checks +are decorated with :func:`~typing.runtime_checkable`. Previously, these checks passed silently. Users should decorate their -subclasses with the :func:`runtime_checkable` decorator +subclasses with the :func:`!runtime_checkable` decorator if they want runtime protocols. (Contributed by Yurii Karabas in :issue:`38908`.) @@ -1595,8 +1595,8 @@ Optimizations :func:`map`, :func:`filter`, :func:`reversed`, :func:`bool` and :func:`float`. (Contributed by Donghee Na and Jeroen Demeyer in :issue:`43575`, :issue:`43287`, :issue:`41922`, :issue:`41873` and :issue:`41870`.) -* :class:`BZ2File` performance is improved by removing internal ``RLock``. - This makes :class:`BZ2File` thread unsafe in the face of multiple simultaneous +* :class:`~bz2.BZ2File` performance is improved by removing internal ``RLock``. + This makes :class:`!BZ2File` thread unsafe in the face of multiple simultaneous readers or writers, just like its equivalent classes in :mod:`gzip` and :mod:`lzma` have always been. (Contributed by Inada Naoki in :issue:`43785`.) @@ -1620,7 +1620,7 @@ Deprecated cleaning up old import semantics that were kept for Python 2.7 compatibility. Specifically, :meth:`!find_loader`/:meth:`!find_module` - (superseded by :meth:`~importlib.abc.Finder.find_spec`), + (superseded by :meth:`~importlib.abc.MetaPathFinder.find_spec`), :meth:`~importlib.abc.Loader.load_module` (superseded by :meth:`~importlib.abc.Loader.exec_module`), :meth:`!module_repr` (which the import system @@ -1647,7 +1647,7 @@ Deprecated :meth:`~importlib.abc.Loader.exec_module` instead. (Contributed by Brett Cannon in :issue:`26131`.) -* :meth:`zimport.zipimporter.load_module` has been deprecated in +* :meth:`!zimport.zipimporter.load_module` has been deprecated in preference for :meth:`~zipimport.zipimporter.exec_module`. (Contributed by Brett Cannon in :issue:`26131`.) @@ -1759,23 +1759,23 @@ Deprecated * The following :mod:`ssl` features have been deprecated since Python 3.6, Python 3.7, or OpenSSL 1.1.0 and will be removed in 3.11: - * :data:`~ssl.OP_NO_SSLv2`, :data:`~ssl.OP_NO_SSLv3`, :data:`~ssl.OP_NO_TLSv1`, - :data:`~ssl.OP_NO_TLSv1_1`, :data:`~ssl.OP_NO_TLSv1_2`, and - :data:`~ssl.OP_NO_TLSv1_3` are replaced by - :attr:`sslSSLContext.minimum_version` and - :attr:`sslSSLContext.maximum_version`. + * :data:`!OP_NO_SSLv2`, :data:`!OP_NO_SSLv3`, :data:`!OP_NO_TLSv1`, + :data:`!OP_NO_TLSv1_1`, :data:`!OP_NO_TLSv1_2`, and + :data:`!OP_NO_TLSv1_3` are replaced by + :attr:`~ssl.SSLContext.minimum_version` and + :attr:`~ssl.SSLContext.maximum_version`. - * :data:`~ssl.PROTOCOL_SSLv2`, :data:`~ssl.PROTOCOL_SSLv3`, - :data:`~ssl.PROTOCOL_SSLv23`, :data:`~ssl.PROTOCOL_TLSv1`, - :data:`~ssl.PROTOCOL_TLSv1_1`, :data:`~ssl.PROTOCOL_TLSv1_2`, and - :const:`~ssl.PROTOCOL_TLS` are deprecated in favor of + * :data:`!PROTOCOL_SSLv2`, :data:`!PROTOCOL_SSLv3`, + :data:`!PROTOCOL_SSLv23`, :data:`!PROTOCOL_TLSv1`, + :data:`!PROTOCOL_TLSv1_1`, :data:`!PROTOCOL_TLSv1_2`, and + :const:`!PROTOCOL_TLS` are deprecated in favor of :const:`~ssl.PROTOCOL_TLS_CLIENT` and :const:`~ssl.PROTOCOL_TLS_SERVER` - * :func:`~ssl.wrap_socket` is replaced by :meth:`ssl.SSLContext.wrap_socket` + * :func:`!wrap_socket` is replaced by :meth:`ssl.SSLContext.wrap_socket` - * :func:`~ssl.match_hostname` + * :func:`!match_hostname` - * :func:`~ssl.RAND_pseudo_bytes`, :func:`~ssl.RAND_egd` + * :func:`!RAND_pseudo_bytes`, :func:`!RAND_egd` * NPN features like :meth:`ssl.SSLSocket.selected_npn_protocol` and :meth:`ssl.SSLContext.set_npn_protocols` are replaced by ALPN. From 133c1a7cdb19dd9317e7607ecf8f4fd4fb5842f6 Mon Sep 17 00:00:00 2001 From: Henrik Tunedal Date: Sun, 28 Apr 2024 23:10:44 +0200 Subject: [PATCH 10/22] gh-118293: Suppress mouse cursor feedback when launching Windows processes with multiprocessing (GH-118315) --- Doc/library/subprocess.rst | 16 +++++++ Lib/multiprocessing/popen_spawn_win32.py | 4 +- Lib/subprocess.py | 2 + ...-04-26-14-23-07.gh-issue-118293.ohhPtW.rst | 2 + Modules/_winapi.c | 48 +++++++++++++++++++ 5 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-04-26-14-23-07.gh-issue-118293.ohhPtW.rst diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 1cd233173e85e1..bd35fda7d79225 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1066,6 +1066,22 @@ The :mod:`subprocess` module exposes the following constants. Specifies that the :attr:`STARTUPINFO.wShowWindow` attribute contains additional information. +.. data:: STARTF_FORCEONFEEDBACK + + A :attr:`STARTUPINFO.dwFlags` parameter to specify that the + *Working in Background* mouse cursor will be displayed while a + process is launching. This is the default behavior for GUI + processes. + + .. versionadded:: 3.13 + +.. data:: STARTF_FORCEOFFFEEDBACK + + A :attr:`STARTUPINFO.dwFlags` parameter to specify that the mouse + cursor will not be changed when launching a process. + + .. versionadded:: 3.13 + .. data:: CREATE_NEW_CONSOLE The new process has a new console, instead of inheriting its parent's diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 49d4c7eea22411..62fb0ddbf91a5d 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -3,6 +3,7 @@ import signal import sys import _winapi +from subprocess import STARTUPINFO, STARTF_FORCEOFFFEEDBACK from .context import reduction, get_spawning_popen, set_spawning_popen from . import spawn @@ -74,7 +75,8 @@ def __init__(self, process_obj): try: hp, ht, pid, tid = _winapi.CreateProcess( python_exe, cmd, - None, None, False, 0, env, None, None) + None, None, False, 0, env, None, + STARTUPINFO(dwFlags=STARTF_FORCEOFFFEEDBACK)) _winapi.CloseHandle(ht) except: _winapi.CloseHandle(rhandle) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index d7c7b45127104f..212fdf5b095511 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -83,6 +83,7 @@ STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, + STARTF_FORCEONFEEDBACK, STARTF_FORCEOFFFEEDBACK, ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, @@ -93,6 +94,7 @@ "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", "STD_ERROR_HANDLE", "SW_HIDE", "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", + "STARTF_FORCEONFEEDBACK", "STARTF_FORCEOFFFEEDBACK", "STARTUPINFO", "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", diff --git a/Misc/NEWS.d/next/Windows/2024-04-26-14-23-07.gh-issue-118293.ohhPtW.rst b/Misc/NEWS.d/next/Windows/2024-04-26-14-23-07.gh-issue-118293.ohhPtW.rst new file mode 100644 index 00000000000000..7383a2b32d7240 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-04-26-14-23-07.gh-issue-118293.ohhPtW.rst @@ -0,0 +1,2 @@ +The ``multiprocessing`` module now passes the ``STARTF_FORCEOFFFEEDBACK`` +flag when spawning processes to tell Windows not to change the mouse cursor. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 57b8bdc7ea2448..23e3c0d87f0319 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -72,9 +72,45 @@ #ifndef STARTF_USESHOWWINDOW #define STARTF_USESHOWWINDOW 0x00000001 #endif +#ifndef STARTF_USESIZE +#define STARTF_USESIZE 0x00000002 +#endif +#ifndef STARTF_USEPOSITION +#define STARTF_USEPOSITION 0x00000004 +#endif +#ifndef STARTF_USECOUNTCHARS +#define STARTF_USECOUNTCHARS 0x00000008 +#endif +#ifndef STARTF_USEFILLATTRIBUTE +#define STARTF_USEFILLATTRIBUTE 0x00000010 +#endif +#ifndef STARTF_RUNFULLSCREEN +#define STARTF_RUNFULLSCREEN 0x00000020 +#endif +#ifndef STARTF_FORCEONFEEDBACK +#define STARTF_FORCEONFEEDBACK 0x00000040 +#endif +#ifndef STARTF_FORCEOFFFEEDBACK +#define STARTF_FORCEOFFFEEDBACK 0x00000080 +#endif #ifndef STARTF_USESTDHANDLES #define STARTF_USESTDHANDLES 0x00000100 #endif +#ifndef STARTF_USEHOTKEY +#define STARTF_USEHOTKEY 0x00000200 +#endif +#ifndef STARTF_TITLEISLINKNAME +#define STARTF_TITLEISLINKNAME 0x00000800 +#endif +#ifndef STARTF_TITLEISAPPID +#define STARTF_TITLEISAPPID 0x00001000 +#endif +#ifndef STARTF_PREVENTPINNING +#define STARTF_PREVENTPINNING 0x00002000 +#endif +#ifndef STARTF_UNTRUSTEDSOURCE +#define STARTF_UNTRUSTEDSOURCE 0x00008000 +#endif typedef struct { PyTypeObject *overlapped_type; @@ -3061,7 +3097,19 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, SEC_RESERVE); WINAPI_CONSTANT(F_DWORD, SEC_WRITECOMBINE); WINAPI_CONSTANT(F_DWORD, STARTF_USESHOWWINDOW); + WINAPI_CONSTANT(F_DWORD, STARTF_USESIZE); + WINAPI_CONSTANT(F_DWORD, STARTF_USEPOSITION); + WINAPI_CONSTANT(F_DWORD, STARTF_USECOUNTCHARS); + WINAPI_CONSTANT(F_DWORD, STARTF_USEFILLATTRIBUTE); + WINAPI_CONSTANT(F_DWORD, STARTF_RUNFULLSCREEN); + WINAPI_CONSTANT(F_DWORD, STARTF_FORCEONFEEDBACK); + WINAPI_CONSTANT(F_DWORD, STARTF_FORCEOFFFEEDBACK); WINAPI_CONSTANT(F_DWORD, STARTF_USESTDHANDLES); + WINAPI_CONSTANT(F_DWORD, STARTF_USEHOTKEY); + WINAPI_CONSTANT(F_DWORD, STARTF_TITLEISLINKNAME); + WINAPI_CONSTANT(F_DWORD, STARTF_TITLEISAPPID); + WINAPI_CONSTANT(F_DWORD, STARTF_PREVENTPINNING); + WINAPI_CONSTANT(F_DWORD, STARTF_UNTRUSTEDSOURCE); WINAPI_CONSTANT(F_DWORD, STD_INPUT_HANDLE); WINAPI_CONSTANT(F_DWORD, STD_OUTPUT_HANDLE); WINAPI_CONSTANT(F_DWORD, STD_ERROR_HANDLE); From aa8f6d2708bce1462544d2e2cae05a2595ffd9ec Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 29 Apr 2024 08:38:46 +0300 Subject: [PATCH 11/22] gh-118374: test_ast: Add ``ctx`` argument to ``ast.Name`` calls (#118375) --- Lib/test/test_ast.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 44bcb9bae1cfde..68024304dfb746 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1353,13 +1353,13 @@ def test_dump_incomplete(self): [], [ast.keyword('a', ast.Constant(None))], [], - [ast.Name('dataclass')], + [ast.Name('dataclass', ctx=ast.Load())], ) self.assertEqual(ast.dump(node), - "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass')])", + "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", ) self.assertEqual(ast.dump(node, annotate_fields=False), - "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass')])", + "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", ) def test_dump_show_empty(self): From ab6eda0ee59587e84cb417dd84452b9c6845434c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 29 Apr 2024 07:54:05 +0100 Subject: [PATCH 12/22] GH-118095: Allow a variant of RESUME_CHECK in tier 2 (GH-118286) --- Include/internal/pycore_uop_ids.h | 206 +++++++++++++------------ Include/internal/pycore_uop_metadata.h | 8 + Python/bytecodes.c | 23 +++ Python/executor_cases.c.h | 27 ++++ Python/optimizer.c | 20 ++- Python/optimizer_cases.c.h | 8 + 6 files changed, 189 insertions(+), 103 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index beb182c436d52a..030321ef4fcb23 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -94,46 +94,47 @@ extern "C" { #define _DYNAMIC_EXIT 343 #define _END_SEND END_SEND #define _ERROR_POP_N 344 +#define _EVAL_BREAKER_EXIT 345 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 345 +#define _FATAL_ERROR 346 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 346 -#define _FOR_ITER_GEN_FRAME 347 -#define _FOR_ITER_TIER_TWO 348 +#define _FOR_ITER 347 +#define _FOR_ITER_GEN_FRAME 348 +#define _FOR_ITER_TIER_TWO 349 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 349 -#define _GUARD_BOTH_INT 350 -#define _GUARD_BOTH_UNICODE 351 -#define _GUARD_BUILTINS_VERSION 352 -#define _GUARD_DORV_NO_DICT 353 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 354 -#define _GUARD_GLOBALS_VERSION 355 -#define _GUARD_IS_FALSE_POP 356 -#define _GUARD_IS_NONE_POP 357 -#define _GUARD_IS_NOT_NONE_POP 358 -#define _GUARD_IS_TRUE_POP 359 -#define _GUARD_KEYS_VERSION 360 -#define _GUARD_NOS_FLOAT 361 -#define _GUARD_NOS_INT 362 -#define _GUARD_NOT_EXHAUSTED_LIST 363 -#define _GUARD_NOT_EXHAUSTED_RANGE 364 -#define _GUARD_NOT_EXHAUSTED_TUPLE 365 -#define _GUARD_TOS_FLOAT 366 -#define _GUARD_TOS_INT 367 -#define _GUARD_TYPE_VERSION 368 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 369 -#define _INIT_CALL_PY_EXACT_ARGS 370 -#define _INIT_CALL_PY_EXACT_ARGS_0 371 -#define _INIT_CALL_PY_EXACT_ARGS_1 372 -#define _INIT_CALL_PY_EXACT_ARGS_2 373 -#define _INIT_CALL_PY_EXACT_ARGS_3 374 -#define _INIT_CALL_PY_EXACT_ARGS_4 375 +#define _GUARD_BOTH_FLOAT 350 +#define _GUARD_BOTH_INT 351 +#define _GUARD_BOTH_UNICODE 352 +#define _GUARD_BUILTINS_VERSION 353 +#define _GUARD_DORV_NO_DICT 354 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 355 +#define _GUARD_GLOBALS_VERSION 356 +#define _GUARD_IS_FALSE_POP 357 +#define _GUARD_IS_NONE_POP 358 +#define _GUARD_IS_NOT_NONE_POP 359 +#define _GUARD_IS_TRUE_POP 360 +#define _GUARD_KEYS_VERSION 361 +#define _GUARD_NOS_FLOAT 362 +#define _GUARD_NOS_INT 363 +#define _GUARD_NOT_EXHAUSTED_LIST 364 +#define _GUARD_NOT_EXHAUSTED_RANGE 365 +#define _GUARD_NOT_EXHAUSTED_TUPLE 366 +#define _GUARD_TOS_FLOAT 367 +#define _GUARD_TOS_INT 368 +#define _GUARD_TYPE_VERSION 369 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 370 +#define _INIT_CALL_PY_EXACT_ARGS 371 +#define _INIT_CALL_PY_EXACT_ARGS_0 372 +#define _INIT_CALL_PY_EXACT_ARGS_1 373 +#define _INIT_CALL_PY_EXACT_ARGS_2 374 +#define _INIT_CALL_PY_EXACT_ARGS_3 375 +#define _INIT_CALL_PY_EXACT_ARGS_4 376 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -150,65 +151,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 376 -#define _IS_NONE 377 +#define _INTERNAL_INCREMENT_OPT_COUNTER 377 +#define _IS_NONE 378 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 378 -#define _ITER_CHECK_RANGE 379 -#define _ITER_CHECK_TUPLE 380 -#define _ITER_JUMP_LIST 381 -#define _ITER_JUMP_RANGE 382 -#define _ITER_JUMP_TUPLE 383 -#define _ITER_NEXT_LIST 384 -#define _ITER_NEXT_RANGE 385 -#define _ITER_NEXT_TUPLE 386 -#define _JUMP_TO_TOP 387 +#define _ITER_CHECK_LIST 379 +#define _ITER_CHECK_RANGE 380 +#define _ITER_CHECK_TUPLE 381 +#define _ITER_JUMP_LIST 382 +#define _ITER_JUMP_RANGE 383 +#define _ITER_JUMP_TUPLE 384 +#define _ITER_NEXT_LIST 385 +#define _ITER_NEXT_RANGE 386 +#define _ITER_NEXT_TUPLE 387 +#define _JUMP_TO_TOP 388 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 388 -#define _LOAD_ATTR_CLASS 389 -#define _LOAD_ATTR_CLASS_0 390 -#define _LOAD_ATTR_CLASS_1 391 +#define _LOAD_ATTR 389 +#define _LOAD_ATTR_CLASS 390 +#define _LOAD_ATTR_CLASS_0 391 +#define _LOAD_ATTR_CLASS_1 392 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 392 -#define _LOAD_ATTR_INSTANCE_VALUE_0 393 -#define _LOAD_ATTR_INSTANCE_VALUE_1 394 -#define _LOAD_ATTR_METHOD_LAZY_DICT 395 -#define _LOAD_ATTR_METHOD_NO_DICT 396 -#define _LOAD_ATTR_METHOD_WITH_VALUES 397 -#define _LOAD_ATTR_MODULE 398 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 399 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 400 +#define _LOAD_ATTR_INSTANCE_VALUE 393 +#define _LOAD_ATTR_INSTANCE_VALUE_0 394 +#define _LOAD_ATTR_INSTANCE_VALUE_1 395 +#define _LOAD_ATTR_METHOD_LAZY_DICT 396 +#define _LOAD_ATTR_METHOD_NO_DICT 397 +#define _LOAD_ATTR_METHOD_WITH_VALUES 398 +#define _LOAD_ATTR_MODULE 399 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 400 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 401 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 401 -#define _LOAD_ATTR_SLOT_0 402 -#define _LOAD_ATTR_SLOT_1 403 -#define _LOAD_ATTR_WITH_HINT 404 +#define _LOAD_ATTR_SLOT 402 +#define _LOAD_ATTR_SLOT_0 403 +#define _LOAD_ATTR_SLOT_1 404 +#define _LOAD_ATTR_WITH_HINT 405 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 405 -#define _LOAD_CONST_INLINE_BORROW 406 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 407 -#define _LOAD_CONST_INLINE_WITH_NULL 408 +#define _LOAD_CONST_INLINE 406 +#define _LOAD_CONST_INLINE_BORROW 407 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 408 +#define _LOAD_CONST_INLINE_WITH_NULL 409 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 409 -#define _LOAD_FAST_0 410 -#define _LOAD_FAST_1 411 -#define _LOAD_FAST_2 412 -#define _LOAD_FAST_3 413 -#define _LOAD_FAST_4 414 -#define _LOAD_FAST_5 415 -#define _LOAD_FAST_6 416 -#define _LOAD_FAST_7 417 +#define _LOAD_FAST 410 +#define _LOAD_FAST_0 411 +#define _LOAD_FAST_1 412 +#define _LOAD_FAST_2 413 +#define _LOAD_FAST_3 414 +#define _LOAD_FAST_4 415 +#define _LOAD_FAST_5 416 +#define _LOAD_FAST_6 417 +#define _LOAD_FAST_7 418 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 418 -#define _LOAD_GLOBAL_BUILTINS 419 -#define _LOAD_GLOBAL_MODULE 420 +#define _LOAD_GLOBAL 419 +#define _LOAD_GLOBAL_BUILTINS 420 +#define _LOAD_GLOBAL_MODULE 421 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -222,50 +223,51 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 421 -#define _POP_JUMP_IF_FALSE 422 -#define _POP_JUMP_IF_TRUE 423 +#define _POP_FRAME 422 +#define _POP_JUMP_IF_FALSE 423 +#define _POP_JUMP_IF_TRUE 424 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 424 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 425 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 425 +#define _PUSH_FRAME 426 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 426 +#define _REPLACE_WITH_TRUE 427 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR -#define _SAVE_RETURN_OFFSET 427 -#define _SEND 428 +#define _SAVE_RETURN_OFFSET 428 +#define _SEND 429 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SIDE_EXIT 429 -#define _START_EXECUTOR 430 -#define _STORE_ATTR 431 -#define _STORE_ATTR_INSTANCE_VALUE 432 -#define _STORE_ATTR_SLOT 433 +#define _SIDE_EXIT 430 +#define _START_EXECUTOR 431 +#define _STORE_ATTR 432 +#define _STORE_ATTR_INSTANCE_VALUE 433 +#define _STORE_ATTR_SLOT 434 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 434 -#define _STORE_FAST_0 435 -#define _STORE_FAST_1 436 -#define _STORE_FAST_2 437 -#define _STORE_FAST_3 438 -#define _STORE_FAST_4 439 -#define _STORE_FAST_5 440 -#define _STORE_FAST_6 441 -#define _STORE_FAST_7 442 +#define _STORE_FAST 435 +#define _STORE_FAST_0 436 +#define _STORE_FAST_1 437 +#define _STORE_FAST_2 438 +#define _STORE_FAST_3 439 +#define _STORE_FAST_4 440 +#define _STORE_FAST_5 441 +#define _STORE_FAST_6 442 +#define _STORE_FAST_7 443 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 443 +#define _STORE_SUBSCR 444 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 444 +#define _TIER2_RESUME_CHECK 445 +#define _TO_BOOL 446 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -275,12 +277,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 445 +#define _UNPACK_SEQUENCE 447 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 445 +#define MAX_UOP_ID 447 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 776728d04bce00..59e690f3aace35 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -253,6 +253,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DEOPT] = 0, [_SIDE_EXIT] = 0, [_ERROR_POP_N] = HAS_ARG_FLAG, + [_TIER2_RESUME_CHECK] = HAS_EXIT_FLAG, + [_EVAL_BREAKER_EXIT] = HAS_ESCAPES_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -336,6 +338,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", + [_EVAL_BREAKER_EXIT] = "_EVAL_BREAKER_EXIT", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", [_FATAL_ERROR] = "_FATAL_ERROR", @@ -481,6 +484,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", [_SWAP] = "_SWAP", + [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", [_TO_BOOL_INT] = "_TO_BOOL_INT", @@ -968,6 +972,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _ERROR_POP_N: return oparg; + case _TIER2_RESUME_CHECK: + return 0; + case _EVAL_BREAKER_EXIT: + return 0; default: return -1; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fe3d61362e6b02..f688856d6909ca 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4258,6 +4258,29 @@ dummy_func( } + /* Special version of RESUME_CHECK that (when paired with _EVAL_BREAKER_EXIT) + * is safe for tier 2. Progress is guaranteed because _EVAL_BREAKER_EXIT calls + * _Py_HandlePending which clears the eval_breaker so that _TIER2_RESUME_CHECK + * will not exit if it is immediately executed again. */ + tier2 op(_TIER2_RESUME_CHECK, (--)) { +#if defined(__EMSCRIPTEN__) + EXIT_IF(_Py_emscripten_signal_clock == 0); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; +#endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + EXIT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); + assert(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + } + + tier2 op(_EVAL_BREAKER_EXIT, (--)) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_HandlePending(tstate) != 0) { + GOTO_UNWIND(); + } + EXIT_TO_TRACE(); + } + // END BYTECODES // } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 280cca1592ae18..2d9acfeea432bc 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4295,4 +4295,31 @@ break; } + case _TIER2_RESUME_CHECK: { + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker & _PY_EVAL_EVENTS_MASK) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + break; + } + + case _EVAL_BREAKER_EXIT: { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_HandlePending(tstate) != 0) { + GOTO_UNWIND(); + } + EXIT_TO_TRACE(); + break; + } + #undef TIER_TWO diff --git a/Python/optimizer.c b/Python/optimizer.c index 02c9b395027791..fcd7d18f2c2e22 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -690,6 +690,12 @@ translate_bytecode_to_trace( break; } + case RESUME: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + break; + default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; @@ -967,7 +973,18 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int32_t target = (int32_t)uop_get_target(inst); if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { if (target != current_jump_target) { - uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? _SIDE_EXIT : _DEOPT; + uint16_t exit_op; + if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + if (opcode == _TIER2_RESUME_CHECK) { + exit_op = _EVAL_BREAKER_EXIT; + } + else { + exit_op = _SIDE_EXIT; + } + } + else { + exit_op = _DEOPT; + } make_exit(&buffer[next_spare], exit_op, target); current_jump_target = target; current_jump = next_spare; @@ -1075,6 +1092,7 @@ sanity_check(_PyExecutorObject *executor) CHECK( opcode == _DEOPT || opcode == _SIDE_EXIT || + opcode == _EVAL_BREAKER_EXIT || opcode == _ERROR_POP_N); if (opcode == _SIDE_EXIT) { CHECK(inst->format == UOP_FORMAT_EXIT); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b1965687701050..4102d00171fbaf 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2145,3 +2145,11 @@ break; } + case _TIER2_RESUME_CHECK: { + break; + } + + case _EVAL_BREAKER_EXIT: { + break; + } + From c7e7bfc4ca26bf90e0d4959e303770fbfc3a3795 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 29 Apr 2024 08:58:57 +0200 Subject: [PATCH 13/22] gh-115119: Detect _decimal dependencies using pkg-config (#115406) pkg-config is supported for libmpdec 4.0.0 and newer. --- Doc/using/configure.rst | 13 ++ ...-02-13-15-31-28.gh-issue-115119.FnQzAW.rst | 2 + Modules/_decimal/_decimal.c | 14 +- configure | 159 +++++++++++++----- configure.ac | 127 +++++++------- 5 files changed, 211 insertions(+), 104 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-02-13-15-31-28.gh-issue-115119.FnQzAW.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 580d31fd422c5a..e662c0dafdb8de 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -389,6 +389,17 @@ Options for third-party dependencies C compiler and linker flags for ``libffi``, used by :mod:`ctypes` module, overriding ``pkg-config``. +.. option:: LIBMPDEC_CFLAGS +.. option:: LIBMPDEC_LIBS + + C compiler and linker flags for ``libmpdec``, used by :mod:`decimal` module, + overriding ``pkg-config``. + + .. note:: + + These environment variables have no effect unless + :option:`--with-system-libmpdec` is specified. + .. option:: LIBLZMA_CFLAGS .. option:: LIBLZMA_LIBS @@ -798,6 +809,8 @@ Libraries options .. versionadded:: 3.3 + .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. + .. option:: --with-readline=readline|editline Designate a backend library for the :mod:`readline` module. diff --git a/Misc/NEWS.d/next/Build/2024-02-13-15-31-28.gh-issue-115119.FnQzAW.rst b/Misc/NEWS.d/next/Build/2024-02-13-15-31-28.gh-issue-115119.FnQzAW.rst new file mode 100644 index 00000000000000..5111d8f4db6b83 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-13-15-31-28.gh-issue-115119.FnQzAW.rst @@ -0,0 +1,2 @@ +:program:`configure` now uses :program:`pkg-config` to detect :mod:`decimal` +dependencies if the :option:`--with-system-libmpdec` option is given. diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index fa425f4f740d31..c105367a270458 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -34,7 +34,19 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_typeobject.h" #include "complexobject.h" -#include "mpdecimal.h" + +#include + +// Reuse config from mpdecimal.h if present. +#if defined(MPD_CONFIG_64) + #ifndef CONFIG_64 + #define CONFIG_64 MPD_CONFIG_64 + #endif +#elif defined(MPD_CONFIG_32) + #ifndef CONFIG_32 + #define CONFIG_32 MPD_CONFIG_32 + #endif +#endif #include // isascii() #include diff --git a/configure b/configure index 78f86d83077eaa..571ab8c882aa68 100755 --- a/configure +++ b/configure @@ -880,6 +880,7 @@ TCLTK_CFLAGS LIBSQLITE3_LIBS LIBSQLITE3_CFLAGS LIBMPDEC_INTERNAL +LIBMPDEC_LIBS LIBMPDEC_CFLAGS MODULE__CTYPES_MALLOC_CLOSURE LIBFFI_LIBS @@ -1148,6 +1149,8 @@ LIBUUID_CFLAGS LIBUUID_LIBS LIBFFI_CFLAGS LIBFFI_LIBS +LIBMPDEC_CFLAGS +LIBMPDEC_LIBS LIBSQLITE3_CFLAGS LIBSQLITE3_LIBS TCLTK_CFLAGS @@ -1969,6 +1972,10 @@ Some influential environment variables: LIBFFI_CFLAGS C compiler flags for LIBFFI, overriding pkg-config LIBFFI_LIBS linker flags for LIBFFI, overriding pkg-config + LIBMPDEC_CFLAGS + C compiler flags for LIBMPDEC, overriding pkg-config + LIBMPDEC_LIBS + linker flags for LIBMPDEC, overriding pkg-config LIBSQLITE3_CFLAGS C compiler flags for LIBSQLITE3, overriding pkg-config LIBSQLITE3_LIBS @@ -14591,27 +14598,91 @@ printf "%s\n" "$with_system_libmpdec" >&6; } if test "x$with_system_libmpdec" = xyes then : - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LDFLAGS=${LIBMPDEC_LDFLAGS-"-lmpdec"} - LIBMPDEC_INTERNAL= - -else $as_nop +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libmpdec" >&5 +printf %s "checking for libmpdec... " >&6; } - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LDFLAGS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" +if test -n "$LIBMPDEC_CFLAGS"; then + pkg_cv_LIBMPDEC_CFLAGS="$LIBMPDEC_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBMPDEC_CFLAGS=`$PKG_CONFIG --cflags "libmpdec" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LIBMPDEC_LIBS"; then + pkg_cv_LIBMPDEC_LIBS="$LIBMPDEC_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBMPDEC_LIBS=`$PKG_CONFIG --libs "libmpdec" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi - if test "x$with_pydebug" = xyes -then : - as_fn_append LIBMPDEC_CFLAGS " -DTEST_COVERAGE" -fi +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no fi + if test $_pkg_short_errors_supported = yes; then + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libmpdec" 2>&1` + else + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libmpdec" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LIBMPDEC_PKG_ERRORS" >&5 + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL= +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL= +else + LIBMPDEC_CFLAGS=$pkg_cv_LIBMPDEC_CFLAGS + LIBMPDEC_LIBS=$pkg_cv_LIBMPDEC_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } +fi +else $as_nop + LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" +fi +# Disable forced inlining in debug builds, see GH-94847 +if test "x$with_pydebug" = xyes +then : + as_fn_append LIBMPDEC_CFLAGS " -DTEST_COVERAGE" +fi # Check whether _decimal should use a coroutine-local or thread-local context { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-decimal-contextvar" >&5 @@ -14636,51 +14707,53 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_decimal_contextvar" >&5 printf "%s\n" "$with_decimal_contextvar" >&6; } -# Check for libmpdec machine flavor -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for decimal libmpdec machine" >&5 +if test "x$with_system_libmpdec" = xno +then : + # Check for libmpdec machine flavor + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for decimal libmpdec machine" >&5 printf %s "checking for decimal libmpdec machine... " >&6; } -case $ac_sys_system in #( + case $ac_sys_system in #( Darwin*) : libmpdec_system=Darwin ;; #( SunOS*) : libmpdec_system=sunos ;; #( *) : libmpdec_system=other - ;; + ;; esac -libmpdec_machine=unknown -if test "$libmpdec_system" = Darwin; then - # universal here means: build libmpdec with the same arch options - # the python interpreter was built with - libmpdec_machine=universal -elif test $ac_cv_sizeof_size_t -eq 8; then - if test "$ac_cv_gcc_asm_for_x64" = yes; then - libmpdec_machine=x64 - elif test "$ac_cv_type___uint128_t" = yes; then - libmpdec_machine=uint128 - else - libmpdec_machine=ansi64 - fi -elif test $ac_cv_sizeof_size_t -eq 4; then - if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - case $CC in #( + libmpdec_machine=unknown + if test "$libmpdec_system" = Darwin; then + # universal here means: build libmpdec with the same arch options + # the python interpreter was built with + libmpdec_machine=universal + elif test $ac_cv_sizeof_size_t -eq 8; then + if test "$ac_cv_gcc_asm_for_x64" = yes; then + libmpdec_machine=x64 + elif test "$ac_cv_type___uint128_t" = yes; then + libmpdec_machine=uint128 + else + libmpdec_machine=ansi64 + fi + elif test $ac_cv_sizeof_size_t -eq 4; then + if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then + case $CC in #( *gcc*) : libmpdec_machine=ppro ;; #( *clang*) : libmpdec_machine=ppro ;; #( *) : libmpdec_machine=ansi32 - ;; + ;; esac - else - libmpdec_machine=ansi32 - fi -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libmpdec_machine" >&5 + else + libmpdec_machine=ansi32 + fi + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libmpdec_machine" >&5 printf "%s\n" "$libmpdec_machine" >&6; } -case $libmpdec_machine in #( + case $libmpdec_machine in #( x64) : as_fn_append LIBMPDEC_CFLAGS " -DCONFIG_64=1 -DASM=1" ;; #( uint128) : @@ -14697,8 +14770,9 @@ case $libmpdec_machine in #( as_fn_append LIBMPDEC_CFLAGS " -DUNIVERSAL=1" ;; #( *) : as_fn_error $? "_decimal: unsupported architecture" "$LINENO" 5 - ;; + ;; esac +fi if test "$have_ipa_pure_const_bug" = yes; then # Some versions of gcc miscompile inline asm: @@ -14717,6 +14791,9 @@ fi + + + if test "$ac_sys_system" = "Emscripten" -a -z "$LIBSQLITE3_CFLAGS" -a -z "$LIBSQLITE3_LIBS" then : @@ -30310,7 +30387,7 @@ fi then : as_fn_append MODULE_BLOCK "MODULE__DECIMAL_CFLAGS=$LIBMPDEC_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__DECIMAL_LDFLAGS=$LIBMPDEC_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__DECIMAL_LDFLAGS=$LIBMPDEC_LIBS$as_nl" fi if test "$py_cv_module__decimal" = yes; then diff --git a/configure.ac b/configure.ac index 719b8d3a9573b9..a2d6b1357efbc9 100644 --- a/configure.ac +++ b/configure.ac @@ -3961,23 +3961,21 @@ AC_ARG_WITH( [with_system_libmpdec="no"]) AC_MSG_RESULT([$with_system_libmpdec]) -AS_VAR_IF([with_system_libmpdec], [yes], [ - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LDFLAGS=${LIBMPDEC_LDFLAGS-"-lmpdec"} - LIBMPDEC_INTERNAL= -], [ - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LDFLAGS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - - dnl Disable forced inlining in debug builds, see GH-94847 - AS_VAR_IF([with_pydebug], [yes], [ - AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DTEST_COVERAGE"]) - ]) -]) - -AC_SUBST([LIBMPDEC_CFLAGS]) -AC_SUBST([LIBMPDEC_INTERNAL]) +AS_VAR_IF( + [with_system_libmpdec], [yes], + [PKG_CHECK_MODULES( + [LIBMPDEC], [libmpdec], [], + [LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL=])], + [LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)"]) + +# Disable forced inlining in debug builds, see GH-94847 +AS_VAR_IF( + [with_pydebug], [yes], + [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DTEST_COVERAGE"])]) # Check whether _decimal should use a coroutine-local or thread-local context AC_MSG_CHECKING([for --with-decimal-contextvar]) @@ -3998,50 +3996,52 @@ fi AC_MSG_RESULT([$with_decimal_contextvar]) -# Check for libmpdec machine flavor -AC_MSG_CHECKING([for decimal libmpdec machine]) -AS_CASE([$ac_sys_system], - [Darwin*], [libmpdec_system=Darwin], - [SunOS*], [libmpdec_system=sunos], - [libmpdec_system=other] -) - -libmpdec_machine=unknown -if test "$libmpdec_system" = Darwin; then - # universal here means: build libmpdec with the same arch options - # the python interpreter was built with - libmpdec_machine=universal -elif test $ac_cv_sizeof_size_t -eq 8; then - if test "$ac_cv_gcc_asm_for_x64" = yes; then - libmpdec_machine=x64 - elif test "$ac_cv_type___uint128_t" = yes; then - libmpdec_machine=uint128 - else - libmpdec_machine=ansi64 - fi -elif test $ac_cv_sizeof_size_t -eq 4; then - if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - AS_CASE([$CC], - [*gcc*], [libmpdec_machine=ppro], - [*clang*], [libmpdec_machine=ppro], - [libmpdec_machine=ansi32] - ) - else - libmpdec_machine=ansi32 - fi -fi -AC_MSG_RESULT([$libmpdec_machine]) - -AS_CASE([$libmpdec_machine], - [x64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DASM=1"])], - [uint128], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1"])], - [ansi64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1"])], - [ppro], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DASM=1 -Wno-unknown-pragmas"])], - [ansi32], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1"])], - [ansi-legacy], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DLEGACY_COMPILER=1"])], - [universal], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DUNIVERSAL=1"])], - [AC_MSG_ERROR([_decimal: unsupported architecture])] -) +AS_VAR_IF( + [with_system_libmpdec], [no], + [# Check for libmpdec machine flavor + AC_MSG_CHECKING([for decimal libmpdec machine]) + AS_CASE([$ac_sys_system], + [Darwin*], [libmpdec_system=Darwin], + [SunOS*], [libmpdec_system=sunos], + [libmpdec_system=other] + ) + + libmpdec_machine=unknown + if test "$libmpdec_system" = Darwin; then + # universal here means: build libmpdec with the same arch options + # the python interpreter was built with + libmpdec_machine=universal + elif test $ac_cv_sizeof_size_t -eq 8; then + if test "$ac_cv_gcc_asm_for_x64" = yes; then + libmpdec_machine=x64 + elif test "$ac_cv_type___uint128_t" = yes; then + libmpdec_machine=uint128 + else + libmpdec_machine=ansi64 + fi + elif test $ac_cv_sizeof_size_t -eq 4; then + if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then + AS_CASE([$CC], + [*gcc*], [libmpdec_machine=ppro], + [*clang*], [libmpdec_machine=ppro], + [libmpdec_machine=ansi32] + ) + else + libmpdec_machine=ansi32 + fi + fi + AC_MSG_RESULT([$libmpdec_machine]) + + AS_CASE([$libmpdec_machine], + [x64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DASM=1"])], + [uint128], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1"])], + [ansi64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1"])], + [ppro], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DASM=1 -Wno-unknown-pragmas"])], + [ansi32], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1"])], + [ansi-legacy], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DLEGACY_COMPILER=1"])], + [universal], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DUNIVERSAL=1"])], + [AC_MSG_ERROR([_decimal: unsupported architecture])] + )]) if test "$have_ipa_pure_const_bug" = yes; then # Some versions of gcc miscompile inline asm: @@ -4056,6 +4056,9 @@ if test "$have_glibc_memmove_bug" = yes; then AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -U_FORTIFY_SOURCE"]) fi +AC_SUBST([LIBMPDEC_CFLAGS]) +AC_SUBST([LIBMPDEC_INTERNAL]) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -7643,7 +7646,7 @@ PY_STDLIB_MOD([_curses_panel], [], [test "$have_panel" != "no"], [$PANEL_CFLAGS $CURSES_CFLAGS], [$PANEL_LIBS $CURSES_LIBS] ) -PY_STDLIB_MOD([_decimal], [], [], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LDFLAGS]) +PY_STDLIB_MOD([_decimal], [], [], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) PY_STDLIB_MOD([_dbm], [test -n "$with_dbmliborder"], [test "$have_dbm" != "no"], [$DBM_CFLAGS], [$DBM_LIBS]) From 375c94c75dd9eaefaddd89a7f704a031441af286 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 29 Apr 2024 01:54:52 -0700 Subject: [PATCH 14/22] gh-107674: Lazy load line number to improve performance of tracing (GH-118127) --- ...-04-20-20-30-15.gh-issue-107674.GZPOP7.rst | 1 + Objects/frameobject.c | 16 +++-- Python/instrumentation.c | 62 ++++++++++++++----- Python/legacy_tracing.c | 1 + 4 files changed, 61 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-20-20-30-15.gh-issue-107674.GZPOP7.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-20-20-30-15.gh-issue-107674.GZPOP7.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-20-20-30-15.gh-issue-107674.GZPOP7.rst new file mode 100644 index 00000000000000..29d16bd7dd6581 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-20-20-30-15.gh-issue-107674.GZPOP7.rst @@ -0,0 +1 @@ +Lazy load frame line number to improve performance of tracing diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 07b7ef3df46a5c..36538b1f6d53fe 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -41,12 +41,20 @@ int PyFrame_GetLineNumber(PyFrameObject *f) { assert(f != NULL); - if (f->f_lineno != 0) { - return f->f_lineno; + if (f->f_lineno == -1) { + // We should calculate it once. If we can't get the line number, + // set f->f_lineno to 0. + f->f_lineno = PyUnstable_InterpreterFrame_GetLine(f->f_frame); + if (f->f_lineno < 0) { + f->f_lineno = 0; + return -1; + } } - else { - return PyUnstable_InterpreterFrame_GetLine(f->f_frame); + + if (f->f_lineno > 0) { + return f->f_lineno; } + return PyUnstable_InterpreterFrame_GetLine(f->f_frame); } static PyObject * diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 71efeff077633d..328a3b1733d604 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -268,14 +268,15 @@ get_events(_Py_GlobalMonitors *m, int tool_id) * 8 bit value. * if line_delta == -128: * line = None # represented as -1 - * elif line_delta == -127: + * elif line_delta == -127 or line_delta == -126: * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); * else: * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; */ #define NO_LINE -128 -#define COMPUTED_LINE -127 +#define COMPUTED_LINE_LINENO_CHANGE -127 +#define COMPUTED_LINE -126 #define OFFSET_SHIFT 4 @@ -302,7 +303,7 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) return -1; } - assert(line_delta == COMPUTED_LINE); + assert(line_delta == COMPUTED_LINE || line_delta == COMPUTED_LINE_LINENO_CHANGE); /* Look it up */ return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); } @@ -1224,18 +1225,26 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; - int line = compute_line(code, i, line_delta); - assert(line >= 0); - assert(prev != NULL); - int prev_index = (int)(prev - _PyCode_CODE(code)); - int prev_line = _Py_Instrumentation_GetLine(code, prev_index); - if (prev_line == line) { - int prev_opcode = _PyCode_CODE(code)[prev_index].op.code; - /* RESUME and INSTRUMENTED_RESUME are needed for the operation of - * instrumentation, so must never be hidden by an INSTRUMENTED_LINE. - */ - if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) { - goto done; + int line = 0; + + if (line_delta == COMPUTED_LINE_LINENO_CHANGE) { + // We know the line number must have changed, don't need to calculate + // the line number for now because we might not need it. + line = -1; + } else { + line = compute_line(code, i, line_delta); + assert(line >= 0); + assert(prev != NULL); + int prev_index = (int)(prev - _PyCode_CODE(code)); + int prev_line = _Py_Instrumentation_GetLine(code, prev_index); + if (prev_line == line) { + int prev_opcode = _PyCode_CODE(code)[prev_index].op.code; + /* RESUME and INSTRUMENTED_RESUME are needed for the operation of + * instrumentation, so must never be hidden by an INSTRUMENTED_LINE. + */ + if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) { + goto done; + } } } @@ -1260,6 +1269,12 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, tstate->tracing++; /* Call c_tracefunc directly, having set the line number. */ Py_INCREF(frame_obj); + if (line == -1 && line_delta > COMPUTED_LINE) { + /* Only assign f_lineno if it's easy to calculate, otherwise + * do lazy calculation by setting the f_lineno to 0. + */ + line = compute_line(code, i, line_delta); + } frame_obj->f_lineno = line; int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_LINE, Py_None); frame_obj->f_lineno = 0; @@ -1276,6 +1291,11 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, if (tools == 0) { goto done; } + + if (line == -1) { + /* Need to calculate the line number now for monitoring events */ + line = compute_line(code, i, line_delta); + } PyObject *line_obj = PyLong_FromLong(line); if (line_obj == NULL) { return -1; @@ -1477,6 +1497,13 @@ initialize_lines(PyCodeObject *code) */ if (line != current_line && line >= 0) { line_data[i].original_opcode = opcode; + if (line_data[i].line_delta == COMPUTED_LINE) { + /* Label this line as a line with a line number change + * which could help the monitoring callback to quickly + * identify the line number change. + */ + line_data[i].line_delta = COMPUTED_LINE_LINENO_CHANGE; + } } else { line_data[i].original_opcode = 0; @@ -1529,6 +1556,11 @@ initialize_lines(PyCodeObject *code) assert(target >= 0); if (line_data[target].line_delta != NO_LINE) { line_data[target].original_opcode = _Py_GetBaseOpcode(code, target); + if (line_data[target].line_delta == COMPUTED_LINE_LINENO_CHANGE) { + // If the line is a jump target, we are not sure if the line + // number changes, so we set it to COMPUTED_LINE. + line_data[target].line_delta = COMPUTED_LINE; + } } } /* Scan exception table */ diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index b5a17405931825..74118030925e3e 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -174,6 +174,7 @@ call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) Py_INCREF(frame); int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, arg); + frame->f_lineno = 0; Py_DECREF(frame); if (err) { return NULL; From 030fcc47fb71719f63d5c6f8c68717250ec3d5cb Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Mon, 29 Apr 2024 18:59:38 +0800 Subject: [PATCH 15/22] Fix typo in Doc/c-api/typeobj.rst (GH-118377) --- Doc/c-api/typeobj.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 1105b943325077..a6a2c437ea4e16 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1381,7 +1381,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) Py_VISIT(Py_TYPE(self)); It is only needed since Python 3.9. To support Python 3.8 and older, this - line must be conditionnal:: + line must be conditional:: #if PY_VERSION_HEX >= 0x03090000 Py_VISIT(Py_TYPE(self)); From 0315521c52bf162bd01d21699f3ac5e5bbe78a77 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Mon, 29 Apr 2024 19:01:03 +0800 Subject: [PATCH 16/22] Fix typo in Doc/howto/timerfd.rst (GH-118376) --- Doc/howto/timerfd.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/timerfd.rst b/Doc/howto/timerfd.rst index 98f0294f9d082d..b5fc06ae8c6810 100644 --- a/Doc/howto/timerfd.rst +++ b/Doc/howto/timerfd.rst @@ -108,7 +108,7 @@ descriptors to wait until the file descriptor is ready for reading: # In 1.5 seconds, 1st timer, 2nd timer and 3rd timer fires at once. # # If a timer file descriptor is signaled more than once since - # the last os.read() call, os.read() returns the nubmer of signaled + # the last os.read() call, os.read() returns the number of signaled # as host order of class bytes. print(f"Signaled events={events}") for fd, event in events: From 98739c9078d57f37df7d7b9d76ad32f7a92381a3 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Mon, 29 Apr 2024 19:02:54 +0800 Subject: [PATCH 17/22] Fix typo in Doc/c-api/exceptions.rst (GH-118371) --- Doc/c-api/exceptions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index d5c25a68c47bd6..499bfb47cc4be5 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -104,7 +104,7 @@ Printing and clearing Similar to :c:func:`PyErr_WriteUnraisable`, but the *format* and subsequent parameters help format the warning message; they have the same meaning and values as in :c:func:`PyUnicode_FromFormat`. - ``PyErr_WriteUnraisable(obj)`` is roughtly equivalent to + ``PyErr_WriteUnraisable(obj)`` is roughly equivalent to ``PyErr_FormatUnraisable("Exception ignored in: %R", obj)``. If *format* is ``NULL``, only the traceback is printed. From 3c39f335cb7972937465560f510f53bc7772ac06 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Mon, 29 Apr 2024 19:15:15 +0800 Subject: [PATCH 18/22] gh-114099: Fix typos in iOS/README.rst (GH-118378) --- iOS/README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iOS/README.rst b/iOS/README.rst index a0ffa5737aae43..96cb00eb2e9bfe 100644 --- a/iOS/README.rst +++ b/iOS/README.rst @@ -224,7 +224,7 @@ content of the two "thin" ``Python.framework`` directories, plus the ``bin`` and $ lipo -create -output module.dylib path/to/x86_64/module.dylib path/to/arm64/module.dylib -* The header files will be indentical on both architectures, except for +* The header files will be identical on both architectures, except for ``pyconfig.h``. Copy all the headers from one platform (say, arm64), rename ``pyconfig.h`` to ``pyconfig-arm64.h``, and copy the ``pyconfig.h`` for the other architecture into the merged header folder as ``pyconfig-x86_64.h``. @@ -355,7 +355,7 @@ pass in command line arguments to configure test suite operation. To work around this limitation, the arguments that would normally be passed as command line arguments are configured as a static string at the start of the XCTest method ``- (void)testPython`` in ``iOSTestbedTests.m``. To pass an argument to the test -suite, add a a string to the ``argv`` defintion. These arguments will be passed +suite, add a a string to the ``argv`` definition. These arguments will be passed to the test suite as if they had been passed to ``python -m test`` at the command line. From 23d0371bb99b1df183c36883e256f82fdf6a4bea Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 29 Apr 2024 14:16:51 +0300 Subject: [PATCH 19/22] Uncomment one grammar test (#118361) --- Lib/test/test_grammar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 8501006b799262..c72f4387108ca8 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -164,7 +164,7 @@ def test_floats(self): x = 3.14 x = 314. x = 0.314 - # XXX x = 000.314 + x = 000.314 x = .314 x = 3e14 x = 3E14 From 44f57a952ea1c25699f19c6cf1fa47cd300e33aa Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 29 Apr 2024 09:29:07 -0600 Subject: [PATCH 20/22] gh-117953: Split Up _PyImport_LoadDynamicModuleWithSpec() (gh-118203) Basically, I've turned most of _PyImport_LoadDynamicModuleWithSpec() into two new functions (_PyImport_GetModInitFunc() and _PyImport_RunModInitFunc()) and moved the rest of it out into _imp_create_dynamic_impl(). There shouldn't be any changes in behavior. This change makes some future changes simpler. This is particularly relevant to potentially calling each module init function in the main interpreter first. Thus the critical part of the PR is the addition of _PyImport_RunModInitFunc(), which is strictly focused on running the init func and validating the result. A later PR will take it a step farther by capturing error information rather than raising exceptions. FWIW, this change also helps readers by clarifying a bit more about what happens when an extension/builtin module is imported. --- Include/internal/pycore_import.h | 5 +- Include/internal/pycore_importdl.h | 11 +- Include/moduleobject.h | 2 +- Python/import.c | 217 +++++++++++++++++------------ Python/importdl.c | 101 ++++++++------ 5 files changed, 192 insertions(+), 144 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 8d7f0543f8d315..b02769903a6f9b 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -29,9 +29,6 @@ extern int _PyImport_FixupBuiltin( const char *name, /* UTF-8 encoded string */ PyObject *modules ); -// We could probably drop this: -extern int _PyImport_FixupExtensionObject(PyObject*, PyObject *, - PyObject *, PyObject *); // Export for many shared extensions, like '_json' PyAPI_FUNC(PyObject*) _PyImport_GetModuleAttr(PyObject *, PyObject *); @@ -55,7 +52,7 @@ struct _import_runtime_state { Only legacy (single-phase init) extension modules are added and only if they support multiple initialization (m_size >- 0) or are imported in the main interpreter. - This is initialized lazily in _PyImport_FixupExtensionObject(). + This is initialized lazily in fix_up_extension() in import.c. Modules are added there and looked up in _imp.find_extension(). */ _Py_hashtable_t *hashtable; } extensions; diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 8bf7c2a48f66be..55c26f2c5a475e 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -40,10 +40,17 @@ extern int _Py_ext_module_loader_info_init_from_spec( struct _Py_ext_module_loader_info *info, PyObject *spec); -extern PyObject *_PyImport_LoadDynamicModuleWithSpec( +struct _Py_ext_module_loader_result { + PyModuleDef *def; + PyObject *module; +}; +extern PyModInitFunction _PyImport_GetModInitFunc( struct _Py_ext_module_loader_info *info, - PyObject *spec, FILE *fp); +extern int _PyImport_RunModInitFunc( + PyModInitFunction p0, + struct _Py_ext_module_loader_info *info, + struct _Py_ext_module_loader_result *p_res); /* Max length of module suffix searched for -- accommodates "module.slb" */ diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 42b87cc4e91012..83f8c2030dbb8f 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -53,7 +53,7 @@ typedef struct PyModuleDef_Base { /* A copy of the module's __dict__ after the first time it was loaded. This is only set/used for legacy modules that do not support multiple initializations. - It is set by _PyImport_FixupExtensionObject(). */ + It is set by fix_up_extension() in import.c. */ PyObject* m_copy; } PyModuleDef_Base; diff --git a/Python/import.c b/Python/import.c index 56011295f95190..f440cd52866b60 100644 --- a/Python/import.c +++ b/Python/import.c @@ -632,44 +632,45 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) (6). first time (not found in _PyRuntime.imports.extensions): A. _imp_create_dynamic_impl() -> import_find_extension() - B. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec() - C. _PyImport_LoadDynamicModuleWithSpec(): load - D. _PyImport_LoadDynamicModuleWithSpec(): call - E. -> PyModule_Create() -> PyModule_Create2() + B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() + C. _PyImport_GetModInitFunc(): load + D. _imp_create_dynamic_impl() -> _PyImport_RunModInitFunc() + E. _PyImport_RunModInitFunc(): call + F. -> PyModule_Create() -> PyModule_Create2() -> PyModule_CreateInitialized() - F. PyModule_CreateInitialized() -> PyModule_New() - G. PyModule_CreateInitialized(): allocate mod->md_state - H. PyModule_CreateInitialized() -> PyModule_AddFunctions() - I. PyModule_CreateInitialized() -> PyModule_SetDocString() - J. PyModule_CreateInitialized(): set mod->md_def - K. : initialize the module, etc. - L. _PyImport_LoadDynamicModuleWithSpec() - -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() - M. _PyImport_LoadDynamicModuleWithSpec(): set def->m_base.m_init - N. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_FixupExtensionObject() - O. _PyImport_FixupExtensionObject() -> update_global_state_for_extension() - P. update_global_state_for_extension(): - copy __dict__ into def->m_base.m_copy - Q. update_global_state_for_extension(): - add it to _PyRuntime.imports.extensions - R. _PyImport_FixupExtensionObject() -> finish_singlephase_extension() - S. finish_singlephase_extension(): - add it to interp->imports.modules_by_index - T. finish_singlephase_extension(): add it to sys.modules - U. _imp_create_dynamic_impl(): set __file__ - - Step (P) is skipped for core modules (sys/builtins). + G. PyModule_CreateInitialized() -> PyModule_New() + H. PyModule_CreateInitialized(): allocate mod->md_state + I. PyModule_CreateInitialized() -> PyModule_AddFunctions() + J. PyModule_CreateInitialized() -> PyModule_SetDocString() + K. PyModule_CreateInitialized(): set mod->md_def + L. : initialize the module, etc. + M. _PyImport_RunModInitFunc(): set def->m_base.m_init + N. _imp_create_dynamic_impl() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + O. _imp_create_dynamic_impl(): set __file__ + P. _imp_create_dynamic_impl() -> update_global_state_for_extension() + Q. update_global_state_for_extension(): + copy __dict__ into def->m_base.m_copy + R. update_global_state_for_extension(): + add it to _PyRuntime.imports.extensions + S. _imp_create_dynamic_impl() -> finish_singlephase_extension() + T. finish_singlephase_extension(): + add it to interp->imports.modules_by_index + U. finish_singlephase_extension(): add it to sys.modules + + Step (Q) is skipped for core modules (sys/builtins). (6). subsequent times (found in _PyRuntime.imports.extensions): A. _imp_create_dynamic_impl() -> import_find_extension() - B. import_find_extension() -> import_add_module() - C. if name in sys.modules: use that module - D. else: + B. import_find_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + C. import_find_extension() -> import_add_module() + D. if name in sys.modules: use that module + E. else: 1. import_add_module() -> PyModule_NewObject() 2. import_add_module(): set it on sys.modules - E. import_find_extension(): copy the "m_copy" dict into __dict__ - F. _imp_create_dynamic_impl() - -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + F. import_find_extension(): copy the "m_copy" dict into __dict__ + G. import_find_extension(): add to modules_by_index (10). (every time): A. noop @@ -678,19 +679,22 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) ...for single-phase init modules, where m_size >= 0: (6). not main interpreter and never loaded there - every time (not found in _PyRuntime.imports.extensions): - A-N. (same as for m_size == -1) - O-Q. (skipped) - R-U. (same as for m_size == -1) + A-O. (same as for m_size == -1) + P-R. (skipped) + S-U. (same as for m_size == -1) (6). main interpreter - first time (not found in _PyRuntime.imports.extensions): - A-O. (same as for m_size == -1) - P. (skipped) - Q-U. (same as for m_size == -1) + A-Q. (same as for m_size == -1) + R. (skipped) + S-U. (same as for m_size == -1) - (6). previously loaded in main interpreter (found in _PyRuntime.imports.extensions): + (6). subsequent times (found in _PyRuntime.imports.extensions): A. _imp_create_dynamic_impl() -> import_find_extension() - B. import_find_extension(): call def->m_base.m_init - C. import_find_extension(): add the module to sys.modules + B. import_find_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + C. import_find_extension(): call def->m_base.m_init (see above) + D. import_find_extension(): add the module to sys.modules + E. import_find_extension(): add to modules_by_index (10). every time: A. noop @@ -1270,7 +1274,7 @@ finish_singlephase_extension(PyThreadState *tstate, PyObject *name, PyObject *modules) { assert(mod != NULL && PyModule_Check(mod)); - assert(def == PyModule_GetDef(mod)); + assert(def == _PyModule_GetDef(mod)); if (_modules_by_index_set(tstate->interp, def, mod) < 0) { return -1; @@ -1285,47 +1289,6 @@ finish_singlephase_extension(PyThreadState *tstate, return 0; } -int -_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, - PyObject *filename, PyObject *modules) -{ - PyThreadState *tstate = _PyThreadState_GET(); - - if (mod == NULL || !PyModule_Check(mod)) { - PyErr_BadInternalCall(); - return -1; - } - PyModuleDef *def = PyModule_GetDef(mod); - if (def == NULL) { - PyErr_BadInternalCall(); - return -1; - } - - /* Only single-phase init extension modules can reach here. */ - assert(is_singlephase(def)); - assert(!is_core_module(tstate->interp, name, filename)); - assert(!is_core_module(tstate->interp, name, name)); - - struct singlephase_global_update singlephase = {0}; - // gh-88216: Extensions and def->m_base.m_copy can be updated - // when the extension module doesn't support sub-interpreters. - if (def->m_size == -1) { - singlephase.m_dict = PyModule_GetDict(mod); - assert(singlephase.m_dict != NULL); - } - if (update_global_state_for_extension( - tstate, filename, name, def, &singlephase) < 0) - { - return -1; - } - - if (finish_singlephase_extension(tstate, mod, def, name, modules) < 0) { - return -1; - } - - return 0; -} - static PyObject * import_find_extension(PyThreadState *tstate, @@ -1514,7 +1477,12 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) } PyObject *mod = import_find_extension(tstate, &info); - if (mod || _PyErr_Occurred(tstate)) { + if (mod != NULL) { + assert(!_PyErr_Occurred(tstate)); + assert(is_singlephase(_PyModule_GetDef(mod))); + goto finally; + } + else if (_PyErr_Occurred(tstate)) { goto finally; } @@ -3900,19 +3868,24 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) /*[clinic end generated code: output=83249b827a4fde77 input=c31b954f4cf4e09d]*/ { PyObject *mod = NULL; - FILE *fp; + PyModuleDef *def = NULL; + PyThreadState *tstate = _PyThreadState_GET(); struct _Py_ext_module_loader_info info; if (_Py_ext_module_loader_info_init_from_spec(&info, spec) < 0) { return NULL; } - PyThreadState *tstate = _PyThreadState_GET(); mod = import_find_extension(tstate, &info); - if (mod != NULL || _PyErr_Occurred(tstate)) { - assert(mod == NULL || !_PyErr_Occurred(tstate)); + if (mod != NULL) { + assert(!_PyErr_Occurred(tstate)); + assert(is_singlephase(_PyModule_GetDef(mod))); goto finally; } + else if (_PyErr_Occurred(tstate)) { + goto finally; + } + /* Otherwise it must be multi-phase init or the first time it's loaded. */ if (PySys_Audit("import", "OOOOO", info.name, info.filename, Py_None, Py_None, Py_None) < 0) @@ -3920,11 +3893,10 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) goto finally; } - /* Is multi-phase init or this is the first time being loaded. */ - /* We would move this (and the fclose() below) into * _PyImport_GetModInitFunc(), but it isn't clear if the intervening * code relies on fp still being open. */ + FILE *fp; if (file != NULL) { fp = _Py_fopen_obj(info.filename, "r"); if (fp == NULL) { @@ -3935,7 +3907,70 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) fp = NULL; } - mod = _PyImport_LoadDynamicModuleWithSpec(&info, spec, fp); + PyModInitFunction p0 = _PyImport_GetModInitFunc(&info, fp); + if (p0 == NULL) { + goto finally; + } + + struct _Py_ext_module_loader_result res; + if (_PyImport_RunModInitFunc(p0, &info, &res) < 0) { + assert(PyErr_Occurred()); + goto finally; + } + + mod = res.module; + res.module = NULL; + def = res.def; + assert(def != NULL); + + if (mod == NULL) { + //assert(!is_singlephase(def)); + mod = PyModule_FromDefAndSpec(def, spec); + if (mod == NULL) { + goto finally; + } + } + else { + assert(is_singlephase(def)); + assert(!is_core_module(tstate->interp, info.name, info.filename)); + assert(!is_core_module(tstate->interp, info.name, info.name)); + + const char *name_buf = PyBytes_AS_STRING(info.name_encoded); + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + Py_CLEAR(mod); + goto finally; + } + + /* Remember pointer to module init function. */ + res.def->m_base.m_init = p0; + + /* Remember the filename as the __file__ attribute */ + if (PyModule_AddObjectRef(mod, "__file__", info.filename) < 0) { + PyErr_Clear(); /* Not important enough to report */ + } + + struct singlephase_global_update singlephase = {0}; + // gh-88216: Extensions and def->m_base.m_copy can be updated + // when the extension module doesn't support sub-interpreters. + if (def->m_size == -1) { + singlephase.m_dict = PyModule_GetDict(mod); + assert(singlephase.m_dict != NULL); + } + if (update_global_state_for_extension( + tstate, info.filename, info.name, def, &singlephase) < 0) + { + Py_CLEAR(mod); + goto finally; + } + + PyObject *modules = get_modules_dict(tstate, true); + if (finish_singlephase_extension( + tstate, mod, def, info.name, modules) < 0) + { + Py_CLEAR(mod); + goto finally; + } + } // XXX Shouldn't this happen in the error cases too. if (fp) { diff --git a/Python/importdl.c b/Python/importdl.c index f2ad95fbbb507d..cc70a6daf5d0f1 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -179,17 +179,12 @@ _Py_ext_module_loader_info_init_from_spec( } -PyObject * -_PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, - PyObject *spec, FILE *fp) +PyModInitFunction +_PyImport_GetModInitFunc(struct _Py_ext_module_loader_info *info, + FILE *fp) { - PyObject *m = NULL; const char *name_buf = PyBytes_AS_STRING(info->name_encoded); - const char *oldcontext; dl_funcptr exportfunc; - PyModInitFunction p0; - PyModuleDef *def; - #ifdef MS_WINDOWS exportfunc = _PyImport_FindSharedFuncptrWindows( info->hook_prefix, name_buf, info->filename, fp); @@ -213,16 +208,29 @@ _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, Py_DECREF(msg); } } - goto error; + return NULL; } - p0 = (PyModInitFunction)exportfunc; + return (PyModInitFunction)exportfunc; +} + +int +_PyImport_RunModInitFunc(PyModInitFunction p0, + struct _Py_ext_module_loader_info *info, + struct _Py_ext_module_loader_result *p_res) +{ + struct _Py_ext_module_loader_result res = {0}; + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); + + /* Call the module init function. */ /* Package context is needed for single-phase init */ - oldcontext = _PyImport_SwapPackageContext(info->newcontext); - m = p0(); + const char *oldcontext = _PyImport_SwapPackageContext(info->newcontext); + PyObject *m = p0(); _PyImport_SwapPackageContext(oldcontext); + /* Validate the result (and populate "res". */ + if (m == NULL) { if (!PyErr_Occurred()) { PyErr_Format( @@ -236,9 +244,13 @@ _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, PyExc_SystemError, "initialization of %s raised unreported exception", name_buf); + /* We would probably be correct to decref m here, + * but we weren't doing so before, + * so we stick with doing nothing. */ m = NULL; goto error; } + if (Py_IS_TYPE(m, NULL)) { /* This can happen when a PyModuleDef is returned without calling * PyModuleDef_Init on it @@ -246,55 +258,52 @@ _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, PyErr_Format(PyExc_SystemError, "init function of %s returned uninitialized object", name_buf); + /* Likewise, decref'ing here makes sense. However, the original + * code has a note about "prevent segfault in DECREF", + * so we play it safe and leave it alone. */ m = NULL; /* prevent segfault in DECREF */ goto error; } - if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { - return PyModule_FromDefAndSpec((PyModuleDef*)m, spec); - } - /* Fall back to single-phase init mechanism */ - - if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { - goto error; + if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { + /* multi-phase init */ + res.def = (PyModuleDef *)m; + /* Run PyModule_FromDefAndSpec() to finish loading the module. */ } - - if (info->hook_prefix == nonascii_prefix) { - /* don't allow legacy init for non-ASCII module names */ + else if (info->hook_prefix == nonascii_prefix) { + /* It should have been multi-phase init? */ + /* Don't allow legacy init for non-ASCII module names. */ PyErr_Format( PyExc_SystemError, "initialization of %s did not return PyModuleDef", name_buf); - goto error; - } - - /* Remember pointer to module init function. */ - def = PyModule_GetDef(m); - if (def == NULL) { - PyErr_Format(PyExc_SystemError, - "initialization of %s did not return an extension " - "module", name_buf); - goto error; - } - def->m_base.m_init = p0; - - /* Remember the filename as the __file__ attribute */ - if (PyModule_AddObjectRef(m, "__file__", info->filename) < 0) { - PyErr_Clear(); /* Not important enough to report */ + Py_DECREF(m); + return -1; } + else { + /* single-phase init (legacy) */ + res.module = m; - PyObject *modules = PyImport_GetModuleDict(); - if (_PyImport_FixupExtensionObject( - m, info->name, info->filename, modules) < 0) - { - goto error; + res.def = PyModule_GetDef(m); + if (res.def == NULL) { + PyErr_Clear(); + PyErr_Format(PyExc_SystemError, + "initialization of %s did not return an extension " + "module", name_buf); + goto error; + } } - return m; + assert(!PyErr_Occurred()); + *p_res = res; + return 0; error: - Py_XDECREF(m); - return NULL; + assert(PyErr_Occurred()); + Py_CLEAR(res.module); + res.def = NULL; + *p_res = res; + return -1; } #endif /* HAVE_DYNAMIC_LOADING */ From 51c70de998ead35674bf4b2b236e9ce8e89d17b4 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 29 Apr 2024 18:50:11 +0300 Subject: [PATCH 21/22] gh-118351: Adapt support.TEST_MODULES_ENABLED for builds without the config variable (GH-118354) --- Lib/test/support/__init__.py | 5 +++-- Lib/test/test_capi/test_run.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index ea4945466cac82..70d2610aa495c7 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1181,8 +1181,9 @@ def requires_limited_api(test): return test -TEST_MODULES_ENABLED = sysconfig.get_config_var('TEST_MODULES') == 'yes' - +# Windows build doesn't support --disable-test-modules feature, so there's no +# 'TEST_MODULES' var in config +TEST_MODULES_ENABLED = (sysconfig.get_config_var('TEST_MODULES') or 'yes') == 'yes' def requires_specialization(test): return unittest.skipUnless( diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py index bc0ca9dbb7f31e..3a390273ec21ad 100644 --- a/Lib/test/test_capi/test_run.py +++ b/Lib/test/test_capi/test_run.py @@ -2,7 +2,7 @@ import unittest from collections import UserDict from test.support import import_helper -from test.support.os_helper import unlink, TESTFN, TESTFN_UNDECODABLE +from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE NULL = None _testcapi = import_helper.import_module('_testcapi') @@ -35,6 +35,7 @@ class CAPITest(unittest.TestCase): def test_run_stringflags(self): # Test PyRun_StringFlags(). + # XXX: fopen() uses different path encoding than Python on Windows. def run(s, *args): return _testcapi.run_stringflags(s, Py_file_input, *args) source = b'a\n' @@ -63,7 +64,7 @@ def run(s, *args): def test_run_fileexflags(self): # Test PyRun_FileExFlags(). - filename = os.fsencode(TESTFN) + filename = os.fsencode(TESTFN if os.name != 'nt' else TESTFN_ASCII) with open(filename, 'wb') as fp: fp.write(b'a\n') self.addCleanup(unlink, filename) @@ -89,6 +90,7 @@ def run(*args): # CRASHES run(UserDict(), dict(a=1)) @unittest.skipUnless(TESTFN_UNDECODABLE, 'only works if there are undecodable paths') + @unittest.skipIf(os.name == 'nt', 'does not work on Windows') def test_run_fileexflags_with_undecodable_filename(self): run = _testcapi.run_fileexflags try: From 444ac0b7a64ff6b6caba9c2731bd33151ce18ad1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 29 Apr 2024 19:30:48 +0300 Subject: [PATCH 22/22] gh-118285: Fix signatures of operator.{attrgetter,itemgetter,methodcaller} instances (GH-118316) * Allow to specify the signature of custom callable instances of extension type by the __text_signature__ attribute. * Specify signatures of operator.attrgetter, operator.itemgetter, and operator.methodcaller instances. --- Lib/inspect.py | 7 ++++++ Lib/operator.py | 10 ++++---- Lib/test/test_inspect/test_inspect.py | 22 ++++++++++++++++++ Lib/test/test_operator.py | 23 +++++++++++++++++++ ...-04-26-14-53-28.gh-issue-118285.A0_pte.rst | 4 ++++ Modules/_operator.c | 15 ++++++++++++ 6 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-26-14-53-28.gh-issue-118285.A0_pte.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 3c346b27b1f06d..1f4216f0389d28 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2692,6 +2692,13 @@ def _signature_from_callable(obj, *, # An object with __call__ call = getattr_static(type(obj), '__call__', None) if call is not None: + try: + text_sig = obj.__text_signature__ + except AttributeError: + pass + else: + if text_sig: + return _signature_fromstr(sigcls, obj, text_sig) call = _descriptor_get(call, obj) return _get_signature_of(call) diff --git a/Lib/operator.py b/Lib/operator.py index 30116c1189a499..02ccdaa13ddb31 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -239,7 +239,7 @@ class attrgetter: """ __slots__ = ('_attrs', '_call') - def __init__(self, attr, *attrs): + def __init__(self, attr, /, *attrs): if not attrs: if not isinstance(attr, str): raise TypeError('attribute name must be a string') @@ -257,7 +257,7 @@ def func(obj): return tuple(getter(obj) for getter in getters) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -276,7 +276,7 @@ class itemgetter: """ __slots__ = ('_items', '_call') - def __init__(self, item, *items): + def __init__(self, item, /, *items): if not items: self._items = (item,) def func(obj): @@ -288,7 +288,7 @@ def func(obj): return tuple(obj[i] for i in items) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -315,7 +315,7 @@ def __init__(self, name, /, *args, **kwargs): self._args = args self._kwargs = kwargs - def __call__(self, obj): + def __call__(self, obj, /): return getattr(obj, self._name)(*self._args, **self._kwargs) def __repr__(self): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 169d1edb706fc3..6b577090bdff68 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -4090,6 +4090,28 @@ class C: ((('a', ..., ..., "positional_or_keyword"),), ...)) + def test_signature_on_callable_objects_with_text_signature_attr(self): + class C: + __text_signature__ = '(a, /, b, c=True)' + def __call__(self, *args, **kwargs): + pass + + self.assertEqual(self.signature(C), ((), ...)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_only"), + ('b', ..., ..., "positional_or_keyword"), + ('c', True, ..., "positional_or_keyword"), + ), + ...)) + + c = C() + c.__text_signature__ = '(x, y)' + self.assertEqual(self.signature(c), + ((('x', ..., ..., "positional_or_keyword"), + ('y', ..., ..., "positional_or_keyword"), + ), + ...)) + def test_signature_on_wrapper(self): class Wrapper: def __call__(self, b): diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 0d34d671563d19..f8eac8dc002636 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -1,4 +1,5 @@ import unittest +import inspect import pickle import sys from decimal import Decimal @@ -602,6 +603,28 @@ def test_dunder_is_original(self): if dunder: self.assertIs(dunder, orig) + def test_attrgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.attrgetter) + self.assertEqual(str(sig), '(attr, /, *attrs)') + sig = inspect.signature(operator.attrgetter('x', 'z', 'y')) + self.assertEqual(str(sig), '(obj, /)') + + def test_itemgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.itemgetter) + self.assertEqual(str(sig), '(item, /, *items)') + sig = inspect.signature(operator.itemgetter(2, 3, 5)) + self.assertEqual(str(sig), '(obj, /)') + + def test_methodcaller_signature(self): + operator = self.module + sig = inspect.signature(operator.methodcaller) + self.assertEqual(str(sig), '(name, /, *args, **kwargs)') + sig = inspect.signature(operator.methodcaller('foo', 2, y=3)) + self.assertEqual(str(sig), '(obj, /)') + + class PyOperatorTestCase(OperatorTestCase, unittest.TestCase): module = py_operator diff --git a/Misc/NEWS.d/next/Library/2024-04-26-14-53-28.gh-issue-118285.A0_pte.rst b/Misc/NEWS.d/next/Library/2024-04-26-14-53-28.gh-issue-118285.A0_pte.rst new file mode 100644 index 00000000000000..6e8f8d368ca5a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-26-14-53-28.gh-issue-118285.A0_pte.rst @@ -0,0 +1,4 @@ +Allow to specify the signature of custom callable instances of extension +type by the :attr:`__text_signature__` attribute. Specify signatures of +:class:`operator.attrgetter`, :class:`operator.itemgetter`, and +:class:`operator.methodcaller` instances. diff --git a/Modules/_operator.c b/Modules/_operator.c index 1f6496d381adac..306d4508f52a68 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -966,6 +966,18 @@ static struct PyMethodDef operator_methods[] = { }; + +static PyObject * +text_signature(PyObject *self, void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("(obj, /)"); +} + +static PyGetSetDef common_getset[] = { + {"__text_signature__", text_signature, (setter)NULL}, + {NULL} +}; + /* itemgetter object **********************************************************/ typedef struct { @@ -1171,6 +1183,7 @@ static PyType_Slot itemgetter_type_slots[] = { {Py_tp_clear, itemgetter_clear}, {Py_tp_methods, itemgetter_methods}, {Py_tp_members, itemgetter_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, itemgetter_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, itemgetter_repr}, @@ -1528,6 +1541,7 @@ static PyType_Slot attrgetter_type_slots[] = { {Py_tp_clear, attrgetter_clear}, {Py_tp_methods, attrgetter_methods}, {Py_tp_members, attrgetter_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, attrgetter_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, attrgetter_repr}, @@ -1863,6 +1877,7 @@ static PyType_Slot methodcaller_type_slots[] = { {Py_tp_clear, methodcaller_clear}, {Py_tp_methods, methodcaller_methods}, {Py_tp_members, methodcaller_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, methodcaller_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, methodcaller_repr},