From 2104bde572aa60f547274fd529312c5708599001 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sun, 24 Nov 2024 07:20:37 -0800 Subject: [PATCH 001/427] GH-127133: Remove ability to nest argument groups & mutually exclusive groups (#127186) --- Doc/library/argparse.rst | 19 ++--- Doc/whatsnew/3.14.rst | 8 ++ Lib/argparse.py | 20 +---- Lib/test/test_argparse.py | 83 ++++--------------- ...-11-23-04-54-42.gh-issue-127133.WMoJjF.rst | 6 ++ 5 files changed, 41 insertions(+), 95 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-23-04-54-42.gh-issue-127133.WMoJjF.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 410b6e11af0dc0..da4071dee34b8c 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1926,11 +1926,10 @@ Argument groups Note that any arguments not in your user-defined groups will end up back in the usual "positional arguments" and "optional arguments" sections. - .. versionchanged:: 3.11 - Calling :meth:`add_argument_group` on an argument group is deprecated. - This feature was never supported and does not always work correctly. - The function exists on the API by accident through inheritance and - will be removed in the future. + .. deprecated-removed:: 3.11 3.14 + Calling :meth:`add_argument_group` on an argument group now raises an + exception. This nesting was never supported, often failed to work + correctly, and was unintentionally exposed through inheritance. .. deprecated:: 3.14 Passing prefix_chars_ to :meth:`add_argument_group` @@ -1993,11 +1992,11 @@ Mutual exclusion --foo FOO foo help --bar BAR bar help - .. versionchanged:: 3.11 - Calling :meth:`add_argument_group` or :meth:`add_mutually_exclusive_group` - on a mutually exclusive group is deprecated. These features were never - supported and do not always work correctly. The functions exist on the - API by accident through inheritance and will be removed in the future. + .. deprecated-removed:: 3.11 3.14 + Calling :meth:`add_argument_group` or :meth:`add_mutually_exclusive_group` + on a mutually exclusive group now raises an exception. This nesting was + never supported, often failed to work correctly, and was unintentionally + exposed through inheritance. Parser defaults diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index bc4ab6789e1676..2c1acb832080fc 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -641,6 +641,14 @@ argparse of :class:`!argparse.BooleanOptionalAction`. They were deprecated since 3.12. +* Calling :meth:`~argparse.ArgumentParser.add_argument_group` on an argument + group, and calling :meth:`~argparse.ArgumentParser.add_argument_group` or + :meth:`~argparse.ArgumentParser.add_mutually_exclusive_group` on a mutually + exclusive group now raise exceptions. This nesting was never supported, + often failed to work correctly, and was unintentionally exposed through + inheritance. This functionality has been deprecated since Python 3.11. + (Contributed by Savannah Ostrowski in :gh:`127186`.) + ast --- diff --git a/Lib/argparse.py b/Lib/argparse.py index f5a7342c2fc355..d24fa72e573d4f 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1709,14 +1709,7 @@ def _remove_action(self, action): self._group_actions.remove(action) def add_argument_group(self, *args, **kwargs): - import warnings - warnings.warn( - "Nesting argument groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_argument_group(*args, **kwargs) - + raise ValueError('argument groups cannot be nested') class _MutuallyExclusiveGroup(_ArgumentGroup): @@ -1737,15 +1730,8 @@ def _remove_action(self, action): self._container._remove_action(action) self._group_actions.remove(action) - def add_mutually_exclusive_group(self, *args, **kwargs): - import warnings - warnings.warn( - "Nesting mutually exclusive groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_mutually_exclusive_group(*args, **kwargs) - + def add_mutually_exclusive_group(self, **kwargs): + raise ValueError('mutually exclusive groups cannot be nested') def _prog_name(prog=None): if prog is not None: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 69243fde5f0f98..488a3a4ed20fac 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2997,6 +2997,13 @@ def test_group_prefix_chars_default(self): self.assertEqual(msg, str(cm.warning)) self.assertEqual(cm.filename, __file__) + def test_nested_argument_group(self): + parser = argparse.ArgumentParser() + g = parser.add_argument_group() + self.assertRaisesRegex(ValueError, + 'argument groups cannot be nested', + g.add_argument_group) + # =================== # Parent parser tests # =================== @@ -3297,6 +3304,14 @@ def test_empty_group(self): with self.assertRaises(ValueError): parser.parse_args(['-h']) + def test_nested_mutex_groups(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument("--spam") + self.assertRaisesRegex(ValueError, + 'mutually exclusive groups cannot be nested', + g.add_mutually_exclusive_group) + class MEMixin(object): def test_failures_when_not_required(self): @@ -3664,55 +3679,6 @@ def get_parser(self, required): -c c help ''' -class TestMutuallyExclusiveNested(MEMixin, TestCase): - - # Nesting mutually exclusive groups is an undocumented feature - # that came about by accident through inheritance and has been - # the source of many bugs. It is deprecated and this test should - # eventually be removed along with it. - - def get_parser(self, required): - parser = ErrorRaisingArgumentParser(prog='PROG') - group = parser.add_mutually_exclusive_group(required=required) - group.add_argument('-a') - group.add_argument('-b') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group2 = group.add_mutually_exclusive_group(required=required) - group2.add_argument('-c') - group2.add_argument('-d') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group3 = group2.add_mutually_exclusive_group(required=required) - group3.add_argument('-e') - group3.add_argument('-f') - return parser - - usage_when_not_required = '''\ - usage: PROG [-h] [-a A | -b B | [-c C | -d D | [-e E | -f F]]] - ''' - usage_when_required = '''\ - usage: PROG [-h] (-a A | -b B | (-c C | -d D | (-e E | -f F))) - ''' - - help = '''\ - - options: - -h, --help show this help message and exit - -a A - -b B - -c C - -d D - -e E - -f F - ''' - - # We are only interested in testing the behavior of format_usage(). - test_failures_when_not_required = None - test_failures_when_required = None - test_successes_when_not_required = None - test_successes_when_required = None - class TestMutuallyExclusiveOptionalOptional(MEMixin, TestCase): def get_parser(self, required=None): @@ -4883,25 +4849,6 @@ def test_all_suppressed_mutex_with_optional_nargs(self): usage = 'usage: PROG [-h]\n' self.assertEqual(parser.format_usage(), usage) - def test_nested_mutex_groups(self): - parser = argparse.ArgumentParser(prog='PROG') - g = parser.add_mutually_exclusive_group() - g.add_argument("--spam") - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - gg = g.add_mutually_exclusive_group() - gg.add_argument("--hax") - gg.add_argument("--hox", help=argparse.SUPPRESS) - gg.add_argument("--hex") - g.add_argument("--eggs") - parser.add_argument("--num") - - usage = textwrap.dedent('''\ - usage: PROG [-h] [--spam SPAM | [--hax HAX | --hex HEX] | --eggs EGGS] - [--num NUM] - ''') - self.assertEqual(parser.format_usage(), usage) - def test_long_mutex_groups_wrap(self): parser = argparse.ArgumentParser(prog='PROG') g = parser.add_mutually_exclusive_group() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-23-04-54-42.gh-issue-127133.WMoJjF.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-23-04-54-42.gh-issue-127133.WMoJjF.rst new file mode 100644 index 00000000000000..56b496bdf1e310 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-23-04-54-42.gh-issue-127133.WMoJjF.rst @@ -0,0 +1,6 @@ +Calling :meth:`argparse.ArgumentParser.add_argument_group` on an argument group, +and calling :meth:`argparse.ArgumentParser.add_argument_group` or +:meth:`argparse.ArgumentParser.add_mutually_exclusive_group` on a mutually +exclusive group now raise exceptions. This nesting was never supported, often +failed to work correctly, and was unintentionally exposed through inheritance. +This functionality has been deprecated since Python 3.11. From 3d8ac48aed6e27c43bf5d037018edcc31d9e66d8 Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Mon, 25 Nov 2024 00:43:25 +0900 Subject: [PATCH 002/427] gh-101100: Fix sphinx warnings of removed opcodes (#127222) --- Doc/whatsnew/3.11.rst | 2 +- Doc/whatsnew/3.4.rst | 2 +- Doc/whatsnew/3.6.rst | 12 ++++++------ Doc/whatsnew/3.7.rst | 4 ++-- Doc/whatsnew/3.8.rst | 10 +++++----- Doc/whatsnew/3.9.rst | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index e5c6d7cd308504..ed41ecd50b0011 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1670,7 +1670,7 @@ Replaced opcodes | | | for each direction | | | | | +------------------------------------+------------------------------------+-----------------------------------------+ -| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup | +| | :opcode:`!SETUP_WITH` | :opcode:`!BEFORE_WITH` | :keyword:`with` block setup | | | :opcode:`!SETUP_ASYNC_WITH` | | | +------------------------------------+------------------------------------+-----------------------------------------+ diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 9d746b378995c3..71b186aeed7359 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -1979,7 +1979,7 @@ Other Improvements now works correctly (previously it silently returned the first python module in the file). (Contributed by Václav Šmilauer in :issue:`16421`.) -* A new opcode, :opcode:`LOAD_CLASSDEREF`, has been added to fix a bug in the +* A new opcode, :opcode:`!LOAD_CLASSDEREF`, has been added to fix a bug in the loading of free variables in class bodies that could be triggered by certain uses of :ref:`__prepare__ `. (Contributed by Benjamin Peterson in :issue:`17853`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 2276fed60c8db3..1fcc5d7cbfb387 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2366,27 +2366,27 @@ There have been several major changes to the :term:`bytecode` in Python 3.6. (Contributed by Demur Rumed with input and reviews from Serhiy Storchaka and Victor Stinner in :issue:`26647` and :issue:`28050`.) -* The new :opcode:`FORMAT_VALUE` and :opcode:`BUILD_STRING` opcodes as part +* The new :opcode:`!FORMAT_VALUE` and :opcode:`BUILD_STRING` opcodes as part of the :ref:`formatted string literal ` implementation. (Contributed by Eric Smith in :issue:`25483` and Serhiy Storchaka in :issue:`27078`.) -* The new :opcode:`BUILD_CONST_KEY_MAP` opcode to optimize the creation +* The new :opcode:`!BUILD_CONST_KEY_MAP` opcode to optimize the creation of dictionaries with constant keys. (Contributed by Serhiy Storchaka in :issue:`27140`.) * The function call opcodes have been heavily reworked for better performance and simpler implementation. - The :opcode:`MAKE_FUNCTION`, :opcode:`CALL_FUNCTION`, - :opcode:`CALL_FUNCTION_KW` and :opcode:`BUILD_MAP_UNPACK_WITH_CALL` opcodes + The :opcode:`MAKE_FUNCTION`, :opcode:`!CALL_FUNCTION`, + :opcode:`!CALL_FUNCTION_KW` and :opcode:`!BUILD_MAP_UNPACK_WITH_CALL` opcodes have been modified, the new :opcode:`CALL_FUNCTION_EX` and - :opcode:`BUILD_TUPLE_UNPACK_WITH_CALL` have been added, and + :opcode:`!BUILD_TUPLE_UNPACK_WITH_CALL` have been added, and ``CALL_FUNCTION_VAR``, ``CALL_FUNCTION_VAR_KW`` and ``MAKE_CLOSURE`` opcodes have been removed. (Contributed by Demur Rumed in :issue:`27095`, and Serhiy Storchaka in :issue:`27213`, :issue:`28257`.) -* The new :opcode:`SETUP_ANNOTATIONS` and :opcode:`STORE_ANNOTATION` opcodes +* The new :opcode:`SETUP_ANNOTATIONS` and :opcode:`!STORE_ANNOTATION` opcodes have been added to support the new :term:`variable annotation` syntax. (Contributed by Ivan Levkivskyi in :issue:`27985`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 2d433ef4759d52..f420fa5c04479b 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2476,10 +2476,10 @@ avoiding possible problems use new functions :c:func:`PySlice_Unpack` and CPython bytecode changes ------------------------ -There are two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD`. +There are two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`!CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) -The :opcode:`STORE_ANNOTATION` opcode has been removed. +The :opcode:`!STORE_ANNOTATION` opcode has been removed. (Contributed by Mark Shannon in :issue:`32550`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index bdc4ca5cab5245..7aca35b2959cd2 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -2152,11 +2152,11 @@ CPython bytecode changes cleaning-up code for :keyword:`break`, :keyword:`continue` and :keyword:`return`. - Removed opcodes :opcode:`BREAK_LOOP`, :opcode:`CONTINUE_LOOP`, - :opcode:`SETUP_LOOP` and :opcode:`SETUP_EXCEPT`. Added new opcodes - :opcode:`ROT_FOUR`, :opcode:`BEGIN_FINALLY`, :opcode:`CALL_FINALLY` and - :opcode:`POP_FINALLY`. Changed the behavior of :opcode:`END_FINALLY` - and :opcode:`WITH_CLEANUP_START`. + Removed opcodes :opcode:`!BREAK_LOOP`, :opcode:`!CONTINUE_LOOP`, + :opcode:`!SETUP_LOOP` and :opcode:`!SETUP_EXCEPT`. Added new opcodes + :opcode:`!ROT_FOUR`, :opcode:`!BEGIN_FINALLY`, :opcode:`!CALL_FINALLY` and + :opcode:`!POP_FINALLY`. Changed the behavior of :opcode:`!END_FINALLY` + and :opcode:`!WITH_CLEANUP_START`. (Contributed by Mark Shannon, Antoine Pitrou and Serhiy Storchaka in :issue:`17611`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 6118b02dd9bd48..b062e6b4c9bca0 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1203,7 +1203,7 @@ Changes in the C API CPython bytecode changes ------------------------ -* The :opcode:`LOAD_ASSERTION_ERROR` opcode was added for handling the +* The :opcode:`!LOAD_ASSERTION_ERROR` opcode was added for handling the :keyword:`assert` statement. Previously, the assert statement would not work correctly if the :exc:`AssertionError` exception was being shadowed. (Contributed by Zackery Spytz in :issue:`34880`.) From e0ef08f5b444950ad9e900b27f5b5dbc706f4459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 24 Nov 2024 17:36:15 +0100 Subject: [PATCH 003/427] gh-122356: restore the position of a file-like object after `zipfile.is_zipfile` (#122397) --- Lib/test/test_zipfile/test_core.py | 10 ++++++++-- Lib/zipfile/__init__.py | 2 ++ .../2024-07-29-15-20-30.gh-issue-122356.wKCmFx.rst | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-07-29-15-20-30.gh-issue-122356.wKCmFx.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 36f7f542872897..c36228c033a414 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1969,10 +1969,16 @@ def test_is_zip_valid_file(self): zip_contents = fp.read() # - passing a file-like object fp = io.BytesIO() - fp.write(zip_contents) + end = fp.write(zip_contents) + self.assertEqual(fp.tell(), end) + mid = end // 2 + fp.seek(mid, 0) self.assertTrue(zipfile.is_zipfile(fp)) - fp.seek(0, 0) + # check that the position is left unchanged after the call + # see: https://github.com/python/cpython/issues/122356 + self.assertEqual(fp.tell(), mid) self.assertTrue(zipfile.is_zipfile(fp)) + self.assertEqual(fp.tell(), mid) def test_non_existent_file_raises_OSError(self): # make sure we don't raise an AttributeError when a partially-constructed diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 08c83cfb760250..6907ae6d5b7464 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -241,7 +241,9 @@ def is_zipfile(filename): result = False try: if hasattr(filename, "read"): + pos = filename.tell() result = _check_zipfile(fp=filename) + filename.seek(pos) else: with open(filename, "rb") as fp: result = _check_zipfile(fp) diff --git a/Misc/NEWS.d/next/Library/2024-07-29-15-20-30.gh-issue-122356.wKCmFx.rst b/Misc/NEWS.d/next/Library/2024-07-29-15-20-30.gh-issue-122356.wKCmFx.rst new file mode 100644 index 00000000000000..0a4632ca975f6b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-29-15-20-30.gh-issue-122356.wKCmFx.rst @@ -0,0 +1,3 @@ +Guarantee that the position of a file-like object passed to +:func:`zipfile.is_zipfile` is left untouched after the call. +Patch by Bénédikt Tran. From 2bb7846cacb342246aada5ed92d323e54c946063 Mon Sep 17 00:00:00 2001 From: da-woods Date: Sun, 24 Nov 2024 17:21:02 +0000 Subject: [PATCH 004/427] Fix macro expansions in critical section docs (#127226) --- Doc/c-api/init.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 970084ce91ab69..ba1c2852f0bd53 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -2470,7 +2470,7 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. { PyCriticalSection2 _py_cs2; - PyCriticalSection_Begin2(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) In the default build, this macro expands to ``{``. @@ -2482,7 +2482,7 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. In the free-threaded build, this macro expands to:: - PyCriticalSection_End2(&_py_cs2); + PyCriticalSection2_End(&_py_cs2); } In the default build, this macro expands to ``}``. From 97b2ceaaaf88a73a45254912a0e972412879ccbf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 24 Nov 2024 19:30:29 +0200 Subject: [PATCH 005/427] gh-127217: Fix pathname2url() for paths starting with multiple slashes on Posix (GH-127218) --- Lib/test/test_urllib.py | 3 +++ Lib/urllib/request.py | 4 ++++ .../Library/2024-11-24-12-41-31.gh-issue-127217.UAXGFr.rst | 2 ++ 3 files changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-11-24-12-41-31.gh-issue-127217.UAXGFr.rst diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 22ef3c648e271d..fe16badc5bc77d 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1458,6 +1458,9 @@ def test_pathname2url_posix(self): fn = urllib.request.pathname2url self.assertEqual(fn('/'), '/') self.assertEqual(fn('/a/b.c'), '/a/b.c') + self.assertEqual(fn('//a/b.c'), '////a/b.c') + self.assertEqual(fn('///a/b.c'), '/////a/b.c') + self.assertEqual(fn('////a/b.c'), '//////a/b.c') self.assertEqual(fn('/a/b%#c'), '/a/b%25%23c') @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 80be65c613e971..9e555432688a5b 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1667,6 +1667,10 @@ def url2pathname(pathname): def pathname2url(pathname): """OS-specific conversion from a file system path to a relative URL of the 'file' scheme; not recommended for general use.""" + if pathname[:2] == '//': + # Add explicitly empty authority to avoid interpreting the path + # as authority. + pathname = '//' + pathname encoding = sys.getfilesystemencoding() errors = sys.getfilesystemencodeerrors() return quote(pathname, encoding=encoding, errors=errors) diff --git a/Misc/NEWS.d/next/Library/2024-11-24-12-41-31.gh-issue-127217.UAXGFr.rst b/Misc/NEWS.d/next/Library/2024-11-24-12-41-31.gh-issue-127217.UAXGFr.rst new file mode 100644 index 00000000000000..3139e33302f378 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-24-12-41-31.gh-issue-127217.UAXGFr.rst @@ -0,0 +1,2 @@ +Fix :func:`urllib.request.pathname2url` for paths starting with multiple +slashes on Posix. From 307c63358681d669ae39e5ecd814bded4a93443a Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 24 Nov 2024 17:33:46 +0000 Subject: [PATCH 006/427] Improve `pathname2url()` and `url2pathname()` docs (#127125) These functions have long sown confusion among Python developers. The existing documentation says they deal with URL path components, but that doesn't fit the evidence on Windows: >>> pathname2url(r'C:\foo') '///C:/foo' >>> pathname2url(r'\\server\share') '////server/share' # or '//server/share' as of quite recently If these were URL path components, they would imply complete URLs like `file://///C:/foo` and `file://////server/share`. Clearly this isn't right. Yet the implementation in `nturl2path` is deliberate, and the `url2pathname()` function correctly inverts it. On non-Windows platforms, the behaviour until quite recently is to simply quote/unquote the path without adding or removing any leading slashes. This behaviour is compatible with *both* interpretations -- 1) the value is a URL path component (existing docs), and 2) the value is everything following `file:` (this commit) The conclusion I draw is that these functions operate on everything after the `file:` prefix, which may include an authority section. This is the only explanation that fits both the Windows and non-Windows behaviour. It's also a better match for the function names. --- Doc/library/urllib.request.rst | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index a093a5083e037b..9055556a3703bb 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -148,9 +148,15 @@ The :mod:`urllib.request` module defines the following functions: .. function:: pathname2url(path) - Convert the pathname *path* from the local syntax for a path to the form used in - the path component of a URL. This does not produce a complete URL. The return - value will already be quoted using the :func:`~urllib.parse.quote` function. + Convert the given local path to a ``file:`` URL. This function uses + :func:`~urllib.parse.quote` function to encode the path. For historical + reasons, the return value omits the ``file:`` scheme prefix. This example + shows the function being used on Windows:: + + >>> from urllib.request import pathname2url + >>> path = 'C:\\Program Files' + >>> 'file:' + pathname2url(path) + 'file:///C:/Program%20Files' .. versionchanged:: 3.14 Windows drive letters are no longer converted to uppercase. @@ -161,11 +167,17 @@ The :mod:`urllib.request` module defines the following functions: found in any position other than the second character. -.. function:: url2pathname(path) +.. function:: url2pathname(url) + + Convert the given ``file:`` URL to a local path. This function uses + :func:`~urllib.parse.unquote` to decode the URL. For historical reasons, + the given value *must* omit the ``file:`` scheme prefix. This example shows + the function being used on Windows:: - Convert the path component *path* from a percent-encoded URL to the local syntax for a - path. This does not accept a complete URL. This function uses - :func:`~urllib.parse.unquote` to decode *path*. + >>> from urllib.request import url2pathname + >>> url = 'file:///C:/Program%20Files' + >>> url2pathname(url.removeprefix('file:')) + 'C:\\Program Files' .. versionchanged:: 3.14 Windows drive letters are no longer converted to uppercase. From 17c16aea66b606d66f71ae9af381bc34d0ef3f5f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sun, 24 Nov 2024 14:42:50 -0800 Subject: [PATCH 007/427] GH-115869: Make jit_stencils.h reproducible (GH-127166) --- ...-11-22-08-46-46.gh-issue-115869.UVLSKd.rst | 1 + Tools/jit/_stencils.py | 3 ++- Tools/jit/_targets.py | 19 +++++++++++++------ Tools/jit/_writer.py | 2 +- Tools/jit/build.py | 2 +- 5 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst diff --git a/Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst b/Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst new file mode 100644 index 00000000000000..9e8a078983f20b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst @@ -0,0 +1 @@ +Make ``jit_stencils.h`` (which is produced during JIT builds) reproducible. diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 61be8fd3bbdf55..ee761a73fa808a 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -202,7 +202,8 @@ def pad(self, alignment: int) -> None: """Pad the stencil to the given alignment.""" offset = len(self.body) padding = -offset % alignment - self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") + if padding: + self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") self.body.extend([0] * padding) def remove_jump(self, *, alignment: int = 1) -> None: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index d8dce0a905c0f8..d23ced19842347 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -61,10 +61,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: args = ["--disassemble", "--reloc", f"{path}"] output = await _llvm.maybe_run("llvm-objdump", args, echo=self.verbose) if output is not None: + # Make sure that full paths don't leak out (for reproducibility): + long, short = str(path), str(path.name) group.code.disassembly.extend( - line.expandtabs().strip() + line.expandtabs().strip().replace(long, short) for line in output.splitlines() - if not line.isspace() ) args = [ "--elf-output-style=JSON", @@ -90,9 +91,6 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: if group.data.body: line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" group.data.disassembly.append(line) - group.process_relocations( - known_symbols=self.known_symbols, alignment=self.alignment - ) return group def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: @@ -122,6 +120,10 @@ async def _compile( f"-I{CPYTHON / 'Tools' / 'jit'}", "-O3", "-c", + # Shorten full absolute file paths in the generated code (like the + # __FILE__ macro and assert failure messages) for reproducibility: + f"-ffile-prefix-map={CPYTHON}=.", + f"-ffile-prefix-map={tempdir}=.", # This debug info isn't necessary, and bloats out the JIT'ed code. # We *may* be able to re-enable this, process it, and JIT it for a # nicer debugging experience... but that needs a lot more research: @@ -167,7 +169,12 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: c.write_text(template.replace("CASE", case)) coro = self._compile(opname, c, work) tasks.append(group.create_task(coro, name=opname)) - return {task.get_name(): task.result() for task in tasks} + stencil_groups = {task.get_name(): task.result() for task in tasks} + for stencil_group in stencil_groups.values(): + stencil_group.process_relocations( + known_symbols=self.known_symbols, alignment=self.alignment + ) + return stencil_groups def build( self, out: pathlib.Path, *, comment: str = "", force: bool = False diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 81a9f08db31703..5588784544ee00 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -77,6 +77,6 @@ def dump( groups: dict[str, _stencils.StencilGroup], symbols: dict[str, int] ) -> typing.Iterator[str]: """Yield a JIT compiler line-by-line as a C header file.""" - for opname, group in sorted(groups.items()): + for opname, group in groups.items(): yield from _dump_stencil(opname, group) yield from _dump_footer(groups, symbols) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 4a23c6f0afa74a..a8cb0f67c36363 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -8,7 +8,7 @@ import _targets if __name__ == "__main__": - comment = f"$ {shlex.join([sys.executable] + sys.argv)}" + comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "target", type=_targets.get_target, help="a PEP 11 target triple to compile for" From c595eae84c7f665eced09ce67a1ad83b7574d6e5 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 25 Nov 2024 08:29:55 +0300 Subject: [PATCH 008/427] gh-127238: adjust error message for sys.set_int_max_str_digits() (#127241) Now it's correct and closer to Python/initconfig.c --- .../2024-11-25-05-15-21.gh-issue-127238.O8wkH-.rst | 1 + Python/sysmodule.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-25-05-15-21.gh-issue-127238.O8wkH-.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-25-05-15-21.gh-issue-127238.O8wkH-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-25-05-15-21.gh-issue-127238.O8wkH-.rst new file mode 100644 index 00000000000000..e8a274fcd31f26 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-25-05-15-21.gh-issue-127238.O8wkH-.rst @@ -0,0 +1 @@ +Correct error message for :func:`sys.set_int_max_str_digits`. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index aaef5aa532412b..6df297f364c5d3 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -4104,7 +4104,7 @@ _PySys_SetIntMaxStrDigits(int maxdigits) { if (maxdigits != 0 && maxdigits < _PY_LONG_MAX_STR_DIGITS_THRESHOLD) { PyErr_Format( - PyExc_ValueError, "maxdigits must be 0 or larger than %d", + PyExc_ValueError, "maxdigits must be >= %d or 0 for unlimited", _PY_LONG_MAX_STR_DIGITS_THRESHOLD); return -1; } From f4f075b3d5a5b0bc1b13cc27ef2a7de8c103fd04 Mon Sep 17 00:00:00 2001 From: Jun Komoda <45822440+junkmd@users.noreply.github.com> Date: Mon, 25 Nov 2024 21:53:17 +0900 Subject: [PATCH 009/427] Replace `:platform:` with `.. availability::` in `socket.ioctl` doc. (GH-127122) In `socket.ioctl`, `:platform:` -> `.. availability::` --- Doc/library/socket.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 73d495c055ff6e..58323ba6514eac 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1596,8 +1596,6 @@ to sockets. .. method:: socket.ioctl(control, option) - :platform: Windows - The :meth:`ioctl` method is a limited interface to the WSAIoctl system interface. Please refer to the `Win32 documentation `_ for more @@ -1609,9 +1607,12 @@ to sockets. Currently only the following control codes are supported: ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + .. availability:: Windows + .. versionchanged:: 3.6 ``SIO_LOOPBACK_FAST_PATH`` was added. + .. method:: socket.listen([backlog]) Enable a server to accept connections. If *backlog* is specified, it must From c7f1e3e150ca181f4b4bd1e5b59d492749f00be6 Mon Sep 17 00:00:00 2001 From: Jun Komoda <45822440+junkmd@users.noreply.github.com> Date: Mon, 25 Nov 2024 22:55:07 +0900 Subject: [PATCH 010/427] gh-127183: Add `_ctypes.CopyComPointer` tests (GH-127184) * Make `create_shelllink_persist` top level function. * Add `CopyComPointerTests`. * Add more tests. * Update tests. * Add assertions for `Release`'s return value. --- .../test_win32_com_foreign_func.py | 132 +++++++++++++++--- 1 file changed, 115 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_ctypes/test_win32_com_foreign_func.py b/Lib/test/test_ctypes/test_win32_com_foreign_func.py index 651c9277d59af9..8d217fc17efa02 100644 --- a/Lib/test/test_ctypes/test_win32_com_foreign_func.py +++ b/Lib/test/test_ctypes/test_win32_com_foreign_func.py @@ -9,7 +9,7 @@ raise unittest.SkipTest("Windows-specific test") -from _ctypes import COMError +from _ctypes import COMError, CopyComPointer from ctypes import HRESULT @@ -78,6 +78,19 @@ def is_equal_guid(guid1, guid2): ) +def create_shelllink_persist(typ): + ppst = typ() + # https://learn.microsoft.com/en-us/windows/win32/api/combaseapi/nf-combaseapi-cocreateinstance + ole32.CoCreateInstance( + byref(CLSID_ShellLink), + None, + CLSCTX_SERVER, + byref(IID_IPersist), + byref(ppst), + ) + return ppst + + class ForeignFunctionsThatWillCallComMethodsTests(unittest.TestCase): def setUp(self): # https://learn.microsoft.com/en-us/windows/win32/api/combaseapi/nf-combaseapi-coinitializeex @@ -88,19 +101,6 @@ def tearDown(self): ole32.CoUninitialize() gc.collect() - @staticmethod - def create_shelllink_persist(typ): - ppst = typ() - # https://learn.microsoft.com/en-us/windows/win32/api/combaseapi/nf-combaseapi-cocreateinstance - ole32.CoCreateInstance( - byref(CLSID_ShellLink), - None, - CLSCTX_SERVER, - byref(IID_IPersist), - byref(ppst), - ) - return ppst - def test_without_paramflags_and_iid(self): class IUnknown(c_void_p): QueryInterface = proto_query_interface() @@ -110,7 +110,7 @@ class IUnknown(c_void_p): class IPersist(IUnknown): GetClassID = proto_get_class_id() - ppst = self.create_shelllink_persist(IPersist) + ppst = create_shelllink_persist(IPersist) clsid = GUID() hr_getclsid = ppst.GetClassID(byref(clsid)) @@ -142,7 +142,7 @@ class IUnknown(c_void_p): class IPersist(IUnknown): GetClassID = proto_get_class_id(((OUT, "pClassID"),)) - ppst = self.create_shelllink_persist(IPersist) + ppst = create_shelllink_persist(IPersist) clsid = ppst.GetClassID() self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) @@ -167,7 +167,7 @@ class IUnknown(c_void_p): class IPersist(IUnknown): GetClassID = proto_get_class_id(((OUT, "pClassID"),), IID_IPersist) - ppst = self.create_shelllink_persist(IPersist) + ppst = create_shelllink_persist(IPersist) clsid = ppst.GetClassID() self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) @@ -184,5 +184,103 @@ class IPersist(IUnknown): self.assertEqual(0, ppst.Release()) +class CopyComPointerTests(unittest.TestCase): + def setUp(self): + ole32.CoInitializeEx(None, COINIT_APARTMENTTHREADED) + + class IUnknown(c_void_p): + QueryInterface = proto_query_interface(None, IID_IUnknown) + AddRef = proto_add_ref() + Release = proto_release() + + class IPersist(IUnknown): + GetClassID = proto_get_class_id(((OUT, "pClassID"),), IID_IPersist) + + self.IUnknown = IUnknown + self.IPersist = IPersist + + def tearDown(self): + ole32.CoUninitialize() + gc.collect() + + def test_both_are_null(self): + src = self.IPersist() + dst = self.IPersist() + + hr = CopyComPointer(src, byref(dst)) + + self.assertEqual(S_OK, hr) + + self.assertIsNone(src.value) + self.assertIsNone(dst.value) + + def test_src_is_nonnull_and_dest_is_null(self): + # The reference count of the COM pointer created by `CoCreateInstance` + # is initially 1. + src = create_shelllink_persist(self.IPersist) + dst = self.IPersist() + + # `CopyComPointer` calls `AddRef` explicitly in the C implementation. + # The refcount of `src` is incremented from 1 to 2 here. + hr = CopyComPointer(src, byref(dst)) + + self.assertEqual(S_OK, hr) + self.assertEqual(src.value, dst.value) + + # This indicates that the refcount was 2 before the `Release` call. + self.assertEqual(1, src.Release()) + + clsid = dst.GetClassID() + self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) + + self.assertEqual(0, dst.Release()) + + def test_src_is_null_and_dest_is_nonnull(self): + src = self.IPersist() + dst_orig = create_shelllink_persist(self.IPersist) + dst = self.IPersist() + CopyComPointer(dst_orig, byref(dst)) + self.assertEqual(1, dst_orig.Release()) + + clsid = dst.GetClassID() + self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) + + # This does NOT affects the refcount of `dst_orig`. + hr = CopyComPointer(src, byref(dst)) + + self.assertEqual(S_OK, hr) + self.assertIsNone(dst.value) + + with self.assertRaises(ValueError): + dst.GetClassID() # NULL COM pointer access + + # This indicates that the refcount was 1 before the `Release` call. + self.assertEqual(0, dst_orig.Release()) + + def test_both_are_nonnull(self): + src = create_shelllink_persist(self.IPersist) + dst_orig = create_shelllink_persist(self.IPersist) + dst = self.IPersist() + CopyComPointer(dst_orig, byref(dst)) + self.assertEqual(1, dst_orig.Release()) + + self.assertEqual(dst.value, dst_orig.value) + self.assertNotEqual(src.value, dst.value) + + hr = CopyComPointer(src, byref(dst)) + + self.assertEqual(S_OK, hr) + self.assertEqual(src.value, dst.value) + self.assertNotEqual(dst.value, dst_orig.value) + + self.assertEqual(1, src.Release()) + + clsid = dst.GetClassID() + self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) + + self.assertEqual(0, dst.Release()) + self.assertEqual(0, dst_orig.Release()) + + if __name__ == '__main__': unittest.main() From d9331f1b16033796a3958f6e0b12626ed7d3e199 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 25 Nov 2024 15:24:33 +0100 Subject: [PATCH 011/427] gh-107954: Document PEP 741 in What's New 3.14 (#127056) --- Doc/c-api/init_config.rst | 4 ++++ Doc/whatsnew/3.14.rst | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 6194d7446c73e4..e621545cffcee3 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -6,6 +6,8 @@ Python Initialization Configuration *********************************** +.. _pyconfig_api: + PyConfig C API ============== @@ -1592,6 +1594,8 @@ The ``__PYVENV_LAUNCHER__`` environment variable is used to set :c:member:`PyConfig.base_executable`. +.. _pyinitconfig_api: + PyInitConfig C API ================== diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2c1acb832080fc..0e4b9eb0cf0b9c 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -65,6 +65,8 @@ Summary -- release highlights .. PEP-sized items next. +* :ref:`PEP 649: deferred evaluation of annotations ` +* :ref:`PEP 741: Python Configuration C API ` New features @@ -172,6 +174,40 @@ Improved error messages ValueError: too many values to unpack (expected 3, got 4) +.. _whatsnew314-pep741: + +PEP 741: Python Configuration C API +----------------------------------- + +Add a :ref:`PyInitConfig C API ` to configure the Python +initialization without relying on C structures and the ability to make +ABI-compatible changes in the future. + +Complete the :pep:`587` :ref:`PyConfig C API ` by adding +:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension +module; feature previously referred to as the “inittab”. + +Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set +the current runtime configuration. + +PEP 587 “Python Initialization Configuration” unified all the ways to configure +the Python initialization. This PEP unifies also the configuration of the +Python preinitialization and the Python initialization in a single API. +Moreover, this PEP only provides a single choice to embed Python, instead of +having two “Python” and “Isolated” choices (PEP 587), to simplify the API +further. + +The lower level PEP 587 PyConfig API remains available for use cases with an +intentionally higher level of coupling to CPython implementation details (such +as emulating the full functionality of CPython’s CLI, including its +configuration mechanisms). + +(Contributed by Victor Stinner in :gh:`107954`.) + +.. seealso:: + :pep:`741`. + + Other language changes ====================== From 3e7ce6e9aed8616c8ce53eaef4279402d3ee38ec Mon Sep 17 00:00:00 2001 From: Sergey Muraviov Date: Mon, 25 Nov 2024 17:56:46 +0300 Subject: [PATCH 012/427] gh-127248: Fix several removed or misnamed files in pythoncore.vcxproj (GH-127249) --- PCbuild/pythoncore.vcxproj | 7 +------ PCbuild/pythoncore.vcxproj.filters | 14 +------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 95552cade52b75..9ebf58ae8a9bc4 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -169,9 +169,7 @@ - - @@ -220,7 +218,6 @@ - @@ -366,14 +363,12 @@ - - @@ -415,7 +410,7 @@ - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 1708cf6e0b3a52..6c76a6ab592a84 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -231,9 +231,6 @@ Include - - Include - Include @@ -342,9 +339,6 @@ Include - - Modules - Parser @@ -447,15 +441,9 @@ Include\cpython - - Include - Include - - Include - Include @@ -903,7 +891,7 @@ Modules\zlib - + Modules\zlib From d3da04bfc91ec065fe587451409102213af0e57c Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 25 Nov 2024 17:24:37 +0000 Subject: [PATCH 013/427] gh-127022: Remove `_PyEvalFramePushAndInit_UnTagged` (gh-127168) The interpreter now handles `_PyStackRef`s pointing to immortal objects without the deferred bit set, so `_PyEvalFramePushAndInit_UnTagged` is no longer necessary. --- Python/ceval.c | 35 ++++------------------------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 2a3938572c1569..eba0f233a81ef3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1800,33 +1800,6 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, return NULL; } -static _PyInterpreterFrame * -_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, _PyStackRef func, - PyObject *locals, PyObject *const* args, - size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) -{ -#if defined(Py_GIL_DISABLED) - size_t kw_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); - size_t total_argcount = argcount + kw_count; - _PyStackRef *tagged_args_buffer = PyMem_Malloc(sizeof(_PyStackRef) * total_argcount); - if (tagged_args_buffer == NULL) { - PyErr_NoMemory(); - return NULL; - } - for (size_t i = 0; i < argcount; i++) { - tagged_args_buffer[i] = PyStackRef_FromPyObjectSteal(args[i]); - } - for (size_t i = 0; i < kw_count; i++) { - tagged_args_buffer[argcount + i] = PyStackRef_FromPyObjectSteal(args[argcount + i]); - } - _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames, previous); - PyMem_Free(tagged_args_buffer); - return res; -#else - return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames, previous); -#endif -} - /* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. Steals references to func, callargs and kwargs. */ @@ -1851,9 +1824,9 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, Py_INCREF(PyTuple_GET_ITEM(callargs, i)); } } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged( + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, func, locals, - newargs, nargs, kwnames, previous + (_PyStackRef const *)newargs, nargs, kwnames, previous ); if (has_dict) { _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); @@ -1888,9 +1861,9 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, Py_INCREF(args[i+argcount]); } } - _PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged( + _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( tstate, PyStackRef_FromPyObjectNew(func), locals, - args, argcount, kwnames, NULL); + (_PyStackRef const *)args, argcount, kwnames, NULL); if (frame == NULL) { return NULL; } From a2ee89968299fc4f0da4b5a4165025b941213ba5 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 25 Nov 2024 20:32:02 +0300 Subject: [PATCH 014/427] gh-127182: Fix `io.StringIO.__setstate__` crash when `None` is the first value (#127219) Co-authored-by: Victor Stinner --- Lib/test/test_io.py | 15 ++++++++++ ...-11-24-14-20-17.gh-issue-127182.WmfY2g.rst | 2 ++ Modules/_io/stringio.c | 30 ++++++++++--------- 3 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-24-14-20-17.gh-issue-127182.WmfY2g.rst diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index aa1b8268592ff7..f1f8ce57668f3b 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1148,6 +1148,21 @@ def test_disallow_instantiation(self): _io = self._io support.check_disallow_instantiation(self, _io._BytesIOBuffer) + def test_stringio_setstate(self): + # gh-127182: Calling __setstate__() with invalid arguments must not crash + obj = self._io.StringIO() + with self.assertRaisesRegex( + TypeError, + 'initial_value must be str or None, not int', + ): + obj.__setstate__((1, '', 0, {})) + + obj.__setstate__((None, '', 0, {})) # should not crash + self.assertEqual(obj.getvalue(), '') + + obj.__setstate__(('', '', 0, {})) + self.assertEqual(obj.getvalue(), '') + class PyIOTest(IOTest): pass diff --git a/Misc/NEWS.d/next/Library/2024-11-24-14-20-17.gh-issue-127182.WmfY2g.rst b/Misc/NEWS.d/next/Library/2024-11-24-14-20-17.gh-issue-127182.WmfY2g.rst new file mode 100644 index 00000000000000..2cc46ca3d33977 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-24-14-20-17.gh-issue-127182.WmfY2g.rst @@ -0,0 +1,2 @@ +Fix :meth:`!io.StringIO.__setstate__` crash, when :const:`None` was passed as +the first value. diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index f558613dc6233c..65e8d97aa8ac19 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -908,23 +908,25 @@ _io_StringIO___setstate___impl(stringio *self, PyObject *state) once by __init__. So we do not take any chance and replace object's buffer completely. */ { - PyObject *item; - Py_UCS4 *buf; - Py_ssize_t bufsize; - - item = PyTuple_GET_ITEM(state, 0); - buf = PyUnicode_AsUCS4Copy(item); - if (buf == NULL) - return NULL; - bufsize = PyUnicode_GET_LENGTH(item); + PyObject *item = PyTuple_GET_ITEM(state, 0); + if (PyUnicode_Check(item)) { + Py_UCS4 *buf = PyUnicode_AsUCS4Copy(item); + if (buf == NULL) + return NULL; + Py_ssize_t bufsize = PyUnicode_GET_LENGTH(item); - if (resize_buffer(self, bufsize) < 0) { + if (resize_buffer(self, bufsize) < 0) { + PyMem_Free(buf); + return NULL; + } + memcpy(self->buf, buf, bufsize * sizeof(Py_UCS4)); PyMem_Free(buf); - return NULL; + self->string_size = bufsize; + } + else { + assert(item == Py_None); + self->string_size = 0; } - memcpy(self->buf, buf, bufsize * sizeof(Py_UCS4)); - PyMem_Free(buf); - self->string_size = bufsize; } /* Set carefully the position value. Alternatively, we could use the seek From 5bb059fe606983814a445e4dcf9e96fd7cb4951a Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Mon, 25 Nov 2024 19:59:20 +0000 Subject: [PATCH 015/427] GH-127236: `pathname2url()`: generate RFC 1738 URL for absolute POSIX path (#127194) When handed an absolute Windows path such as `C:\foo` or `//server/share`, the `urllib.request.pathname2url()` function returns a URL with an authority section, such as `///C:/foo` or `//server/share` (or before GH-126205, `////server/share`). Only the `file:` prefix is omitted. But when handed an absolute POSIX path such as `/etc/hosts`, or a Windows path of the same form (rooted but lacking a drive), the function returns a URL without an authority section, such as `/etc/hosts`. This patch corrects the discrepancy by adding a `//` prefix before drive-less, rooted paths when generating URLs. --- Doc/library/urllib.request.rst | 10 ++++++---- Lib/nturl2path.py | 20 +++++++++++-------- Lib/test/test_urllib.py | 10 +++++----- Lib/urllib/request.py | 8 +++++--- ...-11-23-12-25-06.gh-issue-125866.wEOP66.rst | 5 +++++ 5 files changed, 33 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-23-12-25-06.gh-issue-125866.wEOP66.rst diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 9055556a3703bb..3c07dc4adf434a 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -159,12 +159,14 @@ The :mod:`urllib.request` module defines the following functions: 'file:///C:/Program%20Files' .. versionchanged:: 3.14 - Windows drive letters are no longer converted to uppercase. + Paths beginning with a slash are converted to URLs with authority + sections. For example, the path ``/etc/hosts`` is converted to + the URL ``///etc/hosts``. .. versionchanged:: 3.14 - On Windows, ``:`` characters not following a drive letter are quoted. In - previous versions, :exc:`OSError` was raised if a colon character was - found in any position other than the second character. + Windows drive letters are no longer converted to uppercase, and ``:`` + characters not following a drive letter no longer cause an + :exc:`OSError` exception to be raised on Windows. .. function:: url2pathname(url) diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py index 01135d1b7683b2..7e13ae3128333d 100644 --- a/Lib/nturl2path.py +++ b/Lib/nturl2path.py @@ -55,13 +55,17 @@ def pathname2url(p): p = p[4:] if p[:4].upper() == 'UNC/': p = '//' + p[4:] - drive, tail = ntpath.splitdrive(p) - if drive[1:] == ':': - # DOS drive specified. Add three slashes to the start, producing - # an authority section with a zero-length authority, and a path - # section starting with a single slash. - drive = f'///{drive}' + drive, root, tail = ntpath.splitroot(p) + if drive: + if drive[1:] == ':': + # DOS drive specified. Add three slashes to the start, producing + # an authority section with a zero-length authority, and a path + # section starting with a single slash. + drive = f'///{drive}' + drive = urllib.parse.quote(drive, safe='/:') + elif root: + # Add explicitly empty authority to path beginning with one slash. + root = f'//{root}' - drive = urllib.parse.quote(drive, safe='/:') tail = urllib.parse.quote(tail) - return drive + tail + return drive + root + tail diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index fe16badc5bc77d..00e46990c406ac 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1434,7 +1434,7 @@ def test_pathname2url_win(self): self.assertEqual(fn('C:\\foo:bar'), '///C:/foo%3Abar') self.assertEqual(fn('foo:bar'), 'foo%3Abar') # No drive letter - self.assertEqual(fn("\\folder\\test\\"), '/folder/test/') + self.assertEqual(fn("\\folder\\test\\"), '///folder/test/') self.assertEqual(fn("\\\\folder\\test\\"), '//folder/test/') self.assertEqual(fn("\\\\\\folder\\test\\"), '///folder/test/') self.assertEqual(fn('\\\\some\\share\\'), '//some/share/') @@ -1447,7 +1447,7 @@ def test_pathname2url_win(self): self.assertEqual(fn('//?/unc/server/share/dir'), '//server/share/dir') # Round-tripping urls = ['///C:', - '/folder/test/', + '///folder/test/', '///C:/foo/bar/spam.foo'] for url in urls: self.assertEqual(fn(urllib.request.url2pathname(url)), url) @@ -1456,12 +1456,12 @@ def test_pathname2url_win(self): 'test specific to POSIX pathnames') def test_pathname2url_posix(self): fn = urllib.request.pathname2url - self.assertEqual(fn('/'), '/') - self.assertEqual(fn('/a/b.c'), '/a/b.c') + self.assertEqual(fn('/'), '///') + self.assertEqual(fn('/a/b.c'), '///a/b.c') self.assertEqual(fn('//a/b.c'), '////a/b.c') self.assertEqual(fn('///a/b.c'), '/////a/b.c') self.assertEqual(fn('////a/b.c'), '//////a/b.c') - self.assertEqual(fn('/a/b%#c'), '/a/b%25%23c') + self.assertEqual(fn('/a/b%#c'), '///a/b%25%23c') @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') def test_pathname2url_nonascii(self): diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 9e555432688a5b..1fcaa89188188d 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1667,9 +1667,11 @@ def url2pathname(pathname): def pathname2url(pathname): """OS-specific conversion from a file system path to a relative URL of the 'file' scheme; not recommended for general use.""" - if pathname[:2] == '//': - # Add explicitly empty authority to avoid interpreting the path - # as authority. + if pathname[:1] == '/': + # Add explicitly empty authority to absolute path. If the path + # starts with exactly one slash then this change is mostly + # cosmetic, but if it begins with two or more slashes then this + # avoids interpreting the path as a URL authority. pathname = '//' + pathname encoding = sys.getfilesystemencoding() errors = sys.getfilesystemencodeerrors() diff --git a/Misc/NEWS.d/next/Library/2024-11-23-12-25-06.gh-issue-125866.wEOP66.rst b/Misc/NEWS.d/next/Library/2024-11-23-12-25-06.gh-issue-125866.wEOP66.rst new file mode 100644 index 00000000000000..0b8ffdb3901db3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-23-12-25-06.gh-issue-125866.wEOP66.rst @@ -0,0 +1,5 @@ +:func:`urllib.request.pathname2url` now adds an empty authority when +generating a URL for a path that begins with exactly one slash. For example, +the path ``/etc/hosts`` is converted to the scheme-less URL ``///etc/hosts``. +As a result of this change, URLs without authorities are only generated for +relative paths. From 26ff32b30553e1f7b0cc822835ad2da8890c180c Mon Sep 17 00:00:00 2001 From: funkyrailroad Date: Mon, 25 Nov 2024 16:34:01 -0500 Subject: [PATCH 016/427] gh-127265: Remove single quotes from 'arrow's in tutorial/errors.rst (GH-127267) --- Doc/tutorial/errors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 4c61cbb2b5bc3d..c01cb8c14a0360 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -23,7 +23,7 @@ complaint you get while you are still learning Python:: ^^^^^ SyntaxError: invalid syntax -The parser repeats the offending line and displays little 'arrow's pointing +The parser repeats the offending line and displays little arrows pointing at the token in the line where the error was detected. The error may be caused by the absence of a token *before* the indicated token. In the example, the error is detected at the function :func:`print`, since a colon From 193890c1ccab4b398a218c6c8e91831477aa2ebb Mon Sep 17 00:00:00 2001 From: mpage Date: Mon, 25 Nov 2024 16:53:49 -0800 Subject: [PATCH 017/427] gh-126612: Include stack effects of uops when computing maximum stack depth (#126894) --- Include/internal/pycore_opcode_metadata.h | 925 ++++++++++++++++++ Lib/test/test_generated_cases.py | 179 +++- Python/flowgraph.c | 66 +- .../opcode_metadata_generator.py | 98 +- Tools/cases_generator/stack.py | 58 +- 5 files changed, 1280 insertions(+), 46 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 53280875f10429..5ce172856e1b19 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -949,6 +949,931 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { #endif +extern int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect); +#ifdef NEED_OPCODE_METADATA +int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { + switch(opcode) { + case BINARY_OP: { + *effect = 0; + return 0; + } + case BINARY_OP_ADD_FLOAT: { + *effect = 0; + return 0; + } + case BINARY_OP_ADD_INT: { + *effect = 0; + return 0; + } + case BINARY_OP_ADD_UNICODE: { + *effect = 0; + return 0; + } + case BINARY_OP_INPLACE_ADD_UNICODE: { + *effect = 0; + return 0; + } + case BINARY_OP_MULTIPLY_FLOAT: { + *effect = 0; + return 0; + } + case BINARY_OP_MULTIPLY_INT: { + *effect = 0; + return 0; + } + case BINARY_OP_SUBTRACT_FLOAT: { + *effect = 0; + return 0; + } + case BINARY_OP_SUBTRACT_INT: { + *effect = 0; + return 0; + } + case BINARY_SLICE: { + *effect = 0; + return 0; + } + case BINARY_SUBSCR: { + *effect = 0; + return 0; + } + case BINARY_SUBSCR_DICT: { + *effect = -1; + return 0; + } + case BINARY_SUBSCR_GETITEM: { + *effect = 0; + return 0; + } + case BINARY_SUBSCR_LIST_INT: { + *effect = -1; + return 0; + } + case BINARY_SUBSCR_STR_INT: { + *effect = -1; + return 0; + } + case BINARY_SUBSCR_TUPLE_INT: { + *effect = -1; + return 0; + } + case BUILD_LIST: { + *effect = 1 - oparg; + return 0; + } + case BUILD_MAP: { + *effect = 1 - oparg*2; + return 0; + } + case BUILD_SET: { + *effect = 1 - oparg; + return 0; + } + case BUILD_SLICE: { + *effect = -1 - ((oparg == 3) ? 1 : 0); + return 0; + } + case BUILD_STRING: { + *effect = 1 - oparg; + return 0; + } + case BUILD_TUPLE: { + *effect = 1 - oparg; + return 0; + } + case CACHE: { + *effect = 0; + return 0; + } + case CALL: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_ALLOC_AND_ENTER_INIT: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_BOUND_METHOD_EXACT_ARGS: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_BOUND_METHOD_GENERAL: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_BUILTIN_CLASS: { + *effect = -1 - oparg; + return 0; + } + case CALL_BUILTIN_FAST: { + *effect = -1 - oparg; + return 0; + } + case CALL_BUILTIN_FAST_WITH_KEYWORDS: { + *effect = -1 - oparg; + return 0; + } + case CALL_BUILTIN_O: { + *effect = -1 - oparg; + return 0; + } + case CALL_FUNCTION_EX: { + *effect = Py_MAX(0, -2 - (oparg & 1)); + return 0; + } + case CALL_INTRINSIC_1: { + *effect = 0; + return 0; + } + case CALL_INTRINSIC_2: { + *effect = -1; + return 0; + } + case CALL_ISINSTANCE: { + *effect = -1 - oparg; + return 0; + } + case CALL_KW: { + int max_eff = Py_MAX(0, -2 - oparg); + max_eff = Py_MAX(max_eff, -3 - oparg); + *effect = max_eff; + return 0; + } + case CALL_KW_BOUND_METHOD: { + int max_eff = Py_MAX(0, -2 - oparg); + max_eff = Py_MAX(max_eff, -3 - oparg); + *effect = max_eff; + return 0; + } + case CALL_KW_NON_PY: { + *effect = Py_MAX(0, -2 - oparg); + return 0; + } + case CALL_KW_PY: { + int max_eff = Py_MAX(0, -2 - oparg); + max_eff = Py_MAX(max_eff, -3 - oparg); + *effect = max_eff; + return 0; + } + case CALL_LEN: { + *effect = -1 - oparg; + return 0; + } + case CALL_LIST_APPEND: { + *effect = -3; + return 0; + } + case CALL_METHOD_DESCRIPTOR_FAST: { + *effect = -1 - oparg; + return 0; + } + case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + *effect = -1 - oparg; + return 0; + } + case CALL_METHOD_DESCRIPTOR_NOARGS: { + *effect = -1 - oparg; + return 0; + } + case CALL_METHOD_DESCRIPTOR_O: { + *effect = -1 - oparg; + return 0; + } + case CALL_NON_PY_GENERAL: { + *effect = Py_MAX(0, -1 - oparg); + return 0; + } + case CALL_PY_EXACT_ARGS: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_PY_GENERAL: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_STR_1: { + *effect = -2; + return 0; + } + case CALL_TUPLE_1: { + *effect = -2; + return 0; + } + case CALL_TYPE_1: { + *effect = -2; + return 0; + } + case CHECK_EG_MATCH: { + *effect = 0; + return 0; + } + case CHECK_EXC_MATCH: { + *effect = 0; + return 0; + } + case CLEANUP_THROW: { + *effect = -1; + return 0; + } + case COMPARE_OP: { + *effect = 0; + return 0; + } + case COMPARE_OP_FLOAT: { + *effect = 0; + return 0; + } + case COMPARE_OP_INT: { + *effect = 0; + return 0; + } + case COMPARE_OP_STR: { + *effect = 0; + return 0; + } + case CONTAINS_OP: { + *effect = 0; + return 0; + } + case CONTAINS_OP_DICT: { + *effect = -1; + return 0; + } + case CONTAINS_OP_SET: { + *effect = -1; + return 0; + } + case CONVERT_VALUE: { + *effect = 0; + return 0; + } + case COPY: { + *effect = 1; + return 0; + } + case COPY_FREE_VARS: { + *effect = 0; + return 0; + } + case DELETE_ATTR: { + *effect = -1; + return 0; + } + case DELETE_DEREF: { + *effect = 0; + return 0; + } + case DELETE_FAST: { + *effect = 0; + return 0; + } + case DELETE_GLOBAL: { + *effect = 0; + return 0; + } + case DELETE_NAME: { + *effect = 0; + return 0; + } + case DELETE_SUBSCR: { + *effect = -2; + return 0; + } + case DICT_MERGE: { + *effect = -1; + return 0; + } + case DICT_UPDATE: { + *effect = -1; + return 0; + } + case END_ASYNC_FOR: { + *effect = -2; + return 0; + } + case END_FOR: { + *effect = -1; + return 0; + } + case END_SEND: { + *effect = -1; + return 0; + } + case ENTER_EXECUTOR: { + *effect = 0; + return 0; + } + case EXIT_INIT_CHECK: { + *effect = -1; + return 0; + } + case EXTENDED_ARG: { + *effect = 0; + return 0; + } + case FORMAT_SIMPLE: { + *effect = 0; + return 0; + } + case FORMAT_WITH_SPEC: { + *effect = -1; + return 0; + } + case FOR_ITER: { + *effect = 1; + return 0; + } + case FOR_ITER_GEN: { + *effect = 1; + return 0; + } + case FOR_ITER_LIST: { + *effect = 1; + return 0; + } + case FOR_ITER_RANGE: { + *effect = 1; + return 0; + } + case FOR_ITER_TUPLE: { + *effect = 1; + return 0; + } + case GET_AITER: { + *effect = 0; + return 0; + } + case GET_ANEXT: { + *effect = 1; + return 0; + } + case GET_AWAITABLE: { + *effect = 0; + return 0; + } + case GET_ITER: { + *effect = 0; + return 0; + } + case GET_LEN: { + *effect = 1; + return 0; + } + case GET_YIELD_FROM_ITER: { + *effect = 0; + return 0; + } + case IMPORT_FROM: { + *effect = 1; + return 0; + } + case IMPORT_NAME: { + *effect = -1; + return 0; + } + case INSTRUMENTED_CALL: { + *effect = Py_MAX(0, -1 - oparg); + return 0; + } + case INSTRUMENTED_CALL_FUNCTION_EX: { + *effect = 0; + return 0; + } + case INSTRUMENTED_CALL_KW: { + *effect = 0; + return 0; + } + case INSTRUMENTED_END_FOR: { + *effect = -1; + return 0; + } + case INSTRUMENTED_END_SEND: { + *effect = -1; + return 0; + } + case INSTRUMENTED_FOR_ITER: { + *effect = 0; + return 0; + } + case INSTRUMENTED_INSTRUCTION: { + *effect = 0; + return 0; + } + case INSTRUMENTED_JUMP_BACKWARD: { + *effect = 0; + return 0; + } + case INSTRUMENTED_JUMP_FORWARD: { + *effect = 0; + return 0; + } + case INSTRUMENTED_LINE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_LOAD_SUPER_ATTR: { + *effect = 0; + return 0; + } + case INSTRUMENTED_POP_JUMP_IF_FALSE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_POP_JUMP_IF_NONE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_POP_JUMP_IF_TRUE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_RESUME: { + *effect = 0; + return 0; + } + case INSTRUMENTED_RETURN_VALUE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_YIELD_VALUE: { + *effect = 0; + return 0; + } + case INTERPRETER_EXIT: { + *effect = -1; + return 0; + } + case IS_OP: { + *effect = -1; + return 0; + } + case JUMP: { + *effect = 0; + return 0; + } + case JUMP_BACKWARD: { + *effect = 0; + return 0; + } + case JUMP_BACKWARD_NO_INTERRUPT: { + *effect = 0; + return 0; + } + case JUMP_FORWARD: { + *effect = 0; + return 0; + } + case JUMP_IF_FALSE: { + *effect = 0; + return 0; + } + case JUMP_IF_TRUE: { + *effect = 0; + return 0; + } + case JUMP_NO_INTERRUPT: { + *effect = 0; + return 0; + } + case LIST_APPEND: { + *effect = -1; + return 0; + } + case LIST_EXTEND: { + *effect = -1; + return 0; + } + case LOAD_ATTR: { + *effect = Py_MAX(1, (oparg & 1)); + return 0; + } + case LOAD_ATTR_CLASS: { + *effect = Py_MAX(0, (oparg & 1)); + return 0; + } + case LOAD_ATTR_CLASS_WITH_METACLASS_CHECK: { + *effect = Py_MAX(0, (oparg & 1)); + return 0; + } + case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: { + *effect = 0; + return 0; + } + case LOAD_ATTR_INSTANCE_VALUE: { + *effect = Py_MAX(0, (oparg & 1)); + return 0; + } + case LOAD_ATTR_METHOD_LAZY_DICT: { + *effect = 1; + return 0; + } + case LOAD_ATTR_METHOD_NO_DICT: { + *effect = 1; + return 0; + } + case LOAD_ATTR_METHOD_WITH_VALUES: { + *effect = 1; + return 0; + } + case LOAD_ATTR_MODULE: { + *effect = Py_MAX(0, (oparg & 1)); + return 0; + } + case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + *effect = 0; + return 0; + } + case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { + *effect = 0; + return 0; + } + case LOAD_ATTR_PROPERTY: { + *effect = 0; + return 0; + } + case LOAD_ATTR_SLOT: { + *effect = Py_MAX(0, (oparg & 1)); + return 0; + } + case LOAD_ATTR_WITH_HINT: { + *effect = Py_MAX(0, (oparg & 1)); + return 0; + } + case LOAD_BUILD_CLASS: { + *effect = 1; + return 0; + } + case LOAD_CLOSURE: { + *effect = 1; + return 0; + } + case LOAD_COMMON_CONSTANT: { + *effect = 1; + return 0; + } + case LOAD_CONST: { + *effect = 1; + return 0; + } + case LOAD_CONST_IMMORTAL: { + *effect = 1; + return 0; + } + case LOAD_DEREF: { + *effect = 1; + return 0; + } + case LOAD_FAST: { + *effect = 1; + return 0; + } + case LOAD_FAST_AND_CLEAR: { + *effect = 1; + return 0; + } + case LOAD_FAST_CHECK: { + *effect = 1; + return 0; + } + case LOAD_FAST_LOAD_FAST: { + *effect = 2; + return 0; + } + case LOAD_FROM_DICT_OR_DEREF: { + *effect = 0; + return 0; + } + case LOAD_FROM_DICT_OR_GLOBALS: { + *effect = 0; + return 0; + } + case LOAD_GLOBAL: { + *effect = Py_MAX(1, 1 + (oparg & 1)); + return 0; + } + case LOAD_GLOBAL_BUILTIN: { + *effect = Py_MAX(1, 1 + (oparg & 1)); + return 0; + } + case LOAD_GLOBAL_MODULE: { + *effect = Py_MAX(1, 1 + (oparg & 1)); + return 0; + } + case LOAD_LOCALS: { + *effect = 1; + return 0; + } + case LOAD_NAME: { + *effect = 1; + return 0; + } + case LOAD_SMALL_INT: { + *effect = 1; + return 0; + } + case LOAD_SPECIAL: { + *effect = 1; + return 0; + } + case LOAD_SUPER_ATTR: { + *effect = Py_MAX(0, -2 + (oparg & 1)); + return 0; + } + case LOAD_SUPER_ATTR_ATTR: { + *effect = -2; + return 0; + } + case LOAD_SUPER_ATTR_METHOD: { + *effect = -1; + return 0; + } + case MAKE_CELL: { + *effect = 0; + return 0; + } + case MAKE_FUNCTION: { + *effect = 0; + return 0; + } + case MAP_ADD: { + *effect = -2; + return 0; + } + case MATCH_CLASS: { + *effect = -2; + return 0; + } + case MATCH_KEYS: { + *effect = 1; + return 0; + } + case MATCH_MAPPING: { + *effect = 1; + return 0; + } + case MATCH_SEQUENCE: { + *effect = 1; + return 0; + } + case NOP: { + *effect = 0; + return 0; + } + case POP_BLOCK: { + *effect = 0; + return 0; + } + case POP_EXCEPT: { + *effect = -1; + return 0; + } + case POP_JUMP_IF_FALSE: { + *effect = -1; + return 0; + } + case POP_JUMP_IF_NONE: { + *effect = 0; + return 0; + } + case POP_JUMP_IF_NOT_NONE: { + *effect = 0; + return 0; + } + case POP_JUMP_IF_TRUE: { + *effect = -1; + return 0; + } + case POP_TOP: { + *effect = -1; + return 0; + } + case PUSH_EXC_INFO: { + *effect = 1; + return 0; + } + case PUSH_NULL: { + *effect = 1; + return 0; + } + case RAISE_VARARGS: { + *effect = -oparg; + return 0; + } + case RERAISE: { + *effect = -1; + return 0; + } + case RESERVED: { + *effect = 0; + return 0; + } + case RESUME: { + *effect = 0; + return 0; + } + case RESUME_CHECK: { + *effect = 0; + return 0; + } + case RETURN_GENERATOR: { + *effect = 1; + return 0; + } + case RETURN_VALUE: { + *effect = 0; + return 0; + } + case SEND: { + *effect = 0; + return 0; + } + case SEND_GEN: { + *effect = 0; + return 0; + } + case SETUP_ANNOTATIONS: { + *effect = 0; + return 0; + } + case SETUP_CLEANUP: { + *effect = 2; + return 0; + } + case SETUP_FINALLY: { + *effect = 1; + return 0; + } + case SETUP_WITH: { + *effect = 1; + return 0; + } + case SET_ADD: { + *effect = -1; + return 0; + } + case SET_FUNCTION_ATTRIBUTE: { + *effect = -1; + return 0; + } + case SET_UPDATE: { + *effect = -1; + return 0; + } + case STORE_ATTR: { + *effect = 0; + return 0; + } + case STORE_ATTR_INSTANCE_VALUE: { + *effect = 0; + return 0; + } + case STORE_ATTR_SLOT: { + *effect = 0; + return 0; + } + case STORE_ATTR_WITH_HINT: { + *effect = 0; + return 0; + } + case STORE_DEREF: { + *effect = -1; + return 0; + } + case STORE_FAST: { + *effect = -1; + return 0; + } + case STORE_FAST_LOAD_FAST: { + *effect = 0; + return 0; + } + case STORE_FAST_MAYBE_NULL: { + *effect = -1; + return 0; + } + case STORE_FAST_STORE_FAST: { + *effect = -2; + return 0; + } + case STORE_GLOBAL: { + *effect = -1; + return 0; + } + case STORE_NAME: { + *effect = -1; + return 0; + } + case STORE_SLICE: { + *effect = 0; + return 0; + } + case STORE_SUBSCR: { + *effect = 0; + return 0; + } + case STORE_SUBSCR_DICT: { + *effect = -3; + return 0; + } + case STORE_SUBSCR_LIST_INT: { + *effect = -3; + return 0; + } + case SWAP: { + *effect = 0; + return 0; + } + case TO_BOOL: { + *effect = 0; + return 0; + } + case TO_BOOL_ALWAYS_TRUE: { + *effect = 0; + return 0; + } + case TO_BOOL_BOOL: { + *effect = 0; + return 0; + } + case TO_BOOL_INT: { + *effect = 0; + return 0; + } + case TO_BOOL_LIST: { + *effect = 0; + return 0; + } + case TO_BOOL_NONE: { + *effect = 0; + return 0; + } + case TO_BOOL_STR: { + *effect = 0; + return 0; + } + case UNARY_INVERT: { + *effect = 0; + return 0; + } + case UNARY_NEGATIVE: { + *effect = 0; + return 0; + } + case UNARY_NOT: { + *effect = 0; + return 0; + } + case UNPACK_EX: { + *effect = (oparg & 0xFF) + (oparg >> 8); + return 0; + } + case UNPACK_SEQUENCE: { + *effect = Py_MAX(1, -1 + oparg); + return 0; + } + case UNPACK_SEQUENCE_LIST: { + *effect = -1 + oparg; + return 0; + } + case UNPACK_SEQUENCE_TUPLE: { + *effect = -1 + oparg; + return 0; + } + case UNPACK_SEQUENCE_TWO_TUPLE: { + *effect = 1; + return 0; + } + case WITH_EXCEPT_START: { + *effect = 1; + return 0; + } + case YIELD_VALUE: { + *effect = 0; + return 0; + } + default: + return -1; + } +} + +#endif + enum InstructionFormat { INSTR_FMT_IB = 1, INSTR_FMT_IBC = 2, diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index ff9a52b7adac8a..66862ec17cca98 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1,9 +1,11 @@ import contextlib import os +import re import sys import tempfile import unittest +from io import StringIO from test import support from test import test_tools @@ -29,10 +31,12 @@ def skip_if_different_mount_drives(): test_tools.skip_if_missing("cases_generator") with test_tools.imports_under_tool("cases_generator"): - from analyzer import StackItem + from analyzer import analyze_forest, StackItem + from cwriter import CWriter import parser from stack import Local, Stack import tier1_generator + import opcode_metadata_generator import optimizer_generator @@ -43,6 +47,14 @@ def handle_stderr(): return support.captured_stderr() +def parse_src(src): + p = parser.Parser(src, "test.c") + nodes = [] + while node := p.definition(): + nodes.append(node) + return nodes + + class TestEffects(unittest.TestCase): def test_effect_sizes(self): stack = Stack() @@ -65,6 +77,171 @@ def test_effect_sizes(self): self.assertEqual(stack.top_offset.to_c(), "1 - oparg - oparg*2 + oparg*4") +class TestGenerateMaxStackEffect(unittest.TestCase): + def check(self, input, output): + analysis = analyze_forest(parse_src(input)) + buf = StringIO() + writer = CWriter(buf, 0, False) + opcode_metadata_generator.generate_max_stack_effect_function(analysis, writer) + buf.seek(0) + generated = buf.read() + matches = re.search(r"(case OP: {[^}]+})", generated) + if matches is None: + self.fail(f"Couldn't find case statement for OP in:\n {generated}") + self.assertEqual(output.strip(), matches.group(1)) + + def test_push_one(self): + input = """ + inst(OP, (a -- b, c)) { + SPAM(); + } + """ + output = """ + case OP: { + *effect = 1; + return 0; + } + """ + self.check(input, output) + + def test_cond_push(self): + input = """ + inst(OP, (a -- b, c if (oparg))) { + SPAM(); + } + """ + output = """ + case OP: { + *effect = ((oparg) ? 1 : 0); + return 0; + } + """ + self.check(input, output) + + def test_ops_pass_two(self): + input = """ + op(A, (-- val1)) { + val1 = SPAM(); + } + op(B, (-- val2)) { + val2 = SPAM(); + } + op(C, (val1, val2 --)) { + } + macro(OP) = A + B + C; + """ + output = """ + case OP: { + *effect = 2; + return 0; + } + """ + self.check(input, output) + + def test_ops_pass_two_cond_push(self): + input = """ + op(A, (-- val1, val2)) { + val1 = 0; + val2 = 1; + } + op(B, (val1, val2 -- val1, val2, val3 if (oparg))) { + val3 = SPAM(); + } + macro(OP) = A + B; + """ + output = """ + case OP: { + *effect = Py_MAX(2, 2 + ((oparg) ? 1 : 0)); + return 0; + } + """ + self.check(input, output) + + def test_pop_push_array(self): + input = """ + inst(OP, (values[oparg] -- values[oparg], above)) { + SPAM(values, oparg); + above = 0; + } + """ + output = """ + case OP: { + *effect = 1; + return 0; + } + """ + self.check(input, output) + + def test_family(self): + input = """ + op(A, (-- val1, val2)) { + val1 = 0; + val2 = 1; + } + op(B, (val1, val2 -- val3)) { + val3 = 2; + } + macro(OP1) = A + B; + + inst(OP, (-- val)) { + val = 0; + } + + family(OP, 0) = { OP1 }; + """ + output = """ + case OP: { + *effect = 2; + return 0; + } + """ + self.check(input, output) + + def test_family_intermediate_array(self): + input = """ + op(A, (-- values[oparg])) { + val1 = 0; + val2 = 1; + } + op(B, (values[oparg] -- val3)) { + val3 = 2; + } + macro(OP1) = A + B; + + inst(OP, (-- val)) { + val = 0; + } + + family(OP, 0) = { OP1 }; + """ + output = """ + case OP: { + *effect = Py_MAX(1, oparg); + return 0; + } + """ + self.check(input, output) + + def test_negative_effect(self): + input = """ + op(A, (val1 -- )) { + } + op(B, (val2 --)) { + } + op(C, (val3 --)) { + } + + macro(OP) = A + B + C; + """ + output = """ + case OP: { + *effect = -1; + return 0; + } + """ + self.check(input, output) + + class TestGeneratedCases(unittest.TestCase): def setUp(self) -> None: super().setUp() diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 5418131950076d..b1097b64469ecd 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -733,7 +733,7 @@ make_cfg_traversal_stack(basicblock *entryblock) { return stack; } -/* Return the stack effect of opcode with argument oparg. +/* Compute the stack effects of opcode with argument oparg. Some opcodes have different stack effect when jump to the target and when not jump. The 'jump' parameter specifies the case: @@ -742,25 +742,42 @@ make_cfg_traversal_stack(basicblock *entryblock) { * 1 -- when jump * -1 -- maximal */ +typedef struct { + /* The stack effect of the instruction. */ + int net; + + /* The maximum stack usage of the instruction. Some instructions may + * temporarily push extra values to the stack while they are executing. + */ + int max; +} stack_effects; + Py_LOCAL(int) -stack_effect(int opcode, int oparg, int jump) +get_stack_effects(int opcode, int oparg, int jump, stack_effects *effects) { if (opcode < 0) { - return PY_INVALID_STACK_EFFECT; + return -1; } if ((opcode <= MAX_REAL_OPCODE) && (_PyOpcode_Deopt[opcode] != opcode)) { // Specialized instructions are not supported. - return PY_INVALID_STACK_EFFECT; + return -1; } int popped = _PyOpcode_num_popped(opcode, oparg); int pushed = _PyOpcode_num_pushed(opcode, oparg); if (popped < 0 || pushed < 0) { - return PY_INVALID_STACK_EFFECT; + return -1; } if (IS_BLOCK_PUSH_OPCODE(opcode) && !jump) { + effects->net = 0; + effects->max = 0; return 0; } - return pushed - popped; + if (_PyOpcode_max_stack_effect(opcode, oparg, &effects->max) < 0) { + return -1; + } + effects->net = pushed - popped; + assert(effects->max >= effects->net); + return 0; } Py_LOCAL_INLINE(int) @@ -807,35 +824,30 @@ calculate_stackdepth(cfg_builder *g) basicblock *next = b->b_next; for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - int effect = stack_effect(instr->i_opcode, instr->i_oparg, 0); - if (effect == PY_INVALID_STACK_EFFECT) { + stack_effects effects; + if (get_stack_effects(instr->i_opcode, instr->i_oparg, 0, &effects) < 0) { PyErr_Format(PyExc_SystemError, "Invalid stack effect for opcode=%d, arg=%i", instr->i_opcode, instr->i_oparg); goto error; } - int new_depth = depth + effect; + int new_depth = depth + effects.net; if (new_depth < 0) { - PyErr_Format(PyExc_ValueError, - "Invalid CFG, stack underflow"); - goto error; - } - if (new_depth > maxdepth) { - maxdepth = new_depth; + PyErr_Format(PyExc_ValueError, + "Invalid CFG, stack underflow"); + goto error; } + maxdepth = Py_MAX(maxdepth, depth + effects.max); if (HAS_TARGET(instr->i_opcode)) { - effect = stack_effect(instr->i_opcode, instr->i_oparg, 1); - if (effect == PY_INVALID_STACK_EFFECT) { + if (get_stack_effects(instr->i_opcode, instr->i_oparg, 1, &effects) < 0) { PyErr_Format(PyExc_SystemError, "Invalid stack effect for opcode=%d, arg=%i", instr->i_opcode, instr->i_oparg); goto error; } - int target_depth = depth + effect; + int target_depth = depth + effects.net; assert(target_depth >= 0); /* invalid code or bug in stackdepth() */ - if (target_depth > maxdepth) { - maxdepth = target_depth; - } + maxdepth = Py_MAX(maxdepth, depth + effects.max); if (stackdepth_push(&sp, instr->i_target, target_depth) < 0) { goto error; } @@ -2936,13 +2948,21 @@ _PyCfg_JumpLabelsToTargets(cfg_builder *g) int PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) { - return stack_effect(opcode, oparg, jump); + stack_effects effs; + if (get_stack_effects(opcode, oparg, jump, &effs) < 0) { + return PY_INVALID_STACK_EFFECT; + } + return effs.net; } int PyCompile_OpcodeStackEffect(int opcode, int oparg) { - return stack_effect(opcode, oparg, -1); + stack_effects effs; + if (get_stack_effects(opcode, oparg, -1, &effs) < 0) { + return PY_INVALID_STACK_EFFECT; + } + return effs.net; } /* Access to compiler optimizations for unit tests. diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 2ad7604af9cc0d..1a9849c0cbbb25 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -19,8 +19,9 @@ cflags, ) from cwriter import CWriter +from dataclasses import dataclass from typing import TextIO -from stack import get_stack_effect +from stack import Stack, get_stack_effect, get_stack_effects # Constants used instead of size for macro expansions. # Note: 1, 2, 4 must match actual cache entry sizes. @@ -107,6 +108,101 @@ def add(inst: Instruction | PseudoInstruction) -> None: emit_stack_effect_function(out, "popped", sorted(popped_data)) emit_stack_effect_function(out, "pushed", sorted(pushed_data)) + generate_max_stack_effect_function(analysis, out) + + +def emit_max_stack_effect_function( + out: CWriter, effects: list[tuple[str, list[str]]] +) -> None: + out.emit("extern int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect);\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit(f"int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) {{\n") + out.emit("switch(opcode) {\n") + for name, exprs in effects: + out.emit(f"case {name}: {{\n") + if len(exprs) == 1: + out.emit(f"*effect = {exprs[0]};\n") + elif len(exprs) == 2: + out.emit(f"*effect = Py_MAX({exprs[0]}, {exprs[1]});\n") + else: + assert len(exprs) > 2 + out.emit(f"int max_eff = Py_MAX({exprs[0]}, {exprs[1]});\n") + for expr in exprs[2:]: + out.emit(f"max_eff = Py_MAX(max_eff, {expr});\n") + out.emit(f"*effect = max_eff;\n") + out.emit(f"return 0;\n") + out.emit("}\n") + out.emit("default:\n") + out.emit(" return -1;\n") + out.emit("}\n") + out.emit("}\n\n") + out.emit("#endif\n\n") + + +@dataclass +class MaxStackEffectSet: + int_effect: int | None + cond_effects: set[str] + + def __init__(self) -> None: + self.int_effect = None + self.cond_effects = set() + + def add(self, stack: Stack) -> None: + top_off = stack.top_offset + top_off_int = top_off.as_int() + if top_off_int is not None: + if self.int_effect is None or top_off_int > self.int_effect: + self.int_effect = top_off_int + else: + self.cond_effects.add(top_off.to_c()) + + def update(self, other: "MaxStackEffectSet") -> None: + if self.int_effect is None: + if other.int_effect is not None: + self.int_effect = other.int_effect + elif other.int_effect is not None: + self.int_effect = max(self.int_effect, other.int_effect) + self.cond_effects.update(other.cond_effects) + + +def generate_max_stack_effect_function(analysis: Analysis, out: CWriter) -> None: + """Generate a function that returns the maximum stack effect of an + instruction while it is executing. + + Specialized instructions that are composed of uops may have a greater stack + effect during instruction execution than the net stack effect of the + instruction if the uops pass values on the stack. + """ + effects: dict[str, MaxStackEffectSet] = {} + + def add(inst: Instruction | PseudoInstruction) -> None: + inst_effect = MaxStackEffectSet() + for stack in get_stack_effects(inst): + inst_effect.add(stack) + effects[inst.name] = inst_effect + + # Collect unique stack effects for each instruction + for inst in analysis.instructions.values(): + add(inst) + for pseudo in analysis.pseudos.values(): + add(pseudo) + + # Merge the effects of all specializations in a family into the generic + # instruction + for family in analysis.families.values(): + for inst in family.members: + effects[family.name].update(effects[inst.name]) + + data: list[tuple[str, list[str]]] = [] + for name, effs in sorted(effects.items(), key=lambda kv: kv[0]): + exprs = [] + if effs.int_effect is not None: + exprs.append(str(effs.int_effect)) + exprs.extend(sorted(effs.cond_effects)) + data.append((name, exprs)) + emit_max_stack_effect_function(out, data) + def generate_is_pseudo(analysis: Analysis, out: CWriter) -> None: """Write the IS_PSEUDO_INSTR macro""" diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index a954bed4df073c..286f47d0cfb11b 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -1,8 +1,9 @@ import re from analyzer import StackItem, StackEffect, Instruction, Uop, PseudoInstruction +from collections import defaultdict from dataclasses import dataclass from cwriter import CWriter -from typing import Iterator +from typing import Iterator, Tuple UNUSED = {"unused"} @@ -385,31 +386,46 @@ def merge(self, other: "Stack", out: CWriter) -> None: self.align(other, out) +def stacks(inst: Instruction | PseudoInstruction) -> Iterator[StackEffect]: + if isinstance(inst, Instruction): + for uop in inst.parts: + if isinstance(uop, Uop): + yield uop.stack + else: + assert isinstance(inst, PseudoInstruction) + yield inst.stack + + +def apply_stack_effect(stack: Stack, effect: StackEffect) -> None: + locals: dict[str, Local] = {} + for var in reversed(effect.inputs): + _, local = stack.pop(var) + if var.name != "unused": + locals[local.name] = local + for var in effect.outputs: + if var.name in locals: + local = locals[var.name] + else: + local = Local.unused(var) + stack.push(local) + + def get_stack_effect(inst: Instruction | PseudoInstruction) -> Stack: stack = Stack() + for s in stacks(inst): + apply_stack_effect(stack, s) + return stack - def stacks(inst: Instruction | PseudoInstruction) -> Iterator[StackEffect]: - if isinstance(inst, Instruction): - for uop in inst.parts: - if isinstance(uop, Uop): - yield uop.stack - else: - assert isinstance(inst, PseudoInstruction) - yield inst.stack +def get_stack_effects(inst: Instruction | PseudoInstruction) -> list[Stack]: + """Returns a list of stack effects after each uop""" + result = [] + stack = Stack() for s in stacks(inst): - locals: dict[str, Local] = {} - for var in reversed(s.inputs): - _, local = stack.pop(var) - if var.name != "unused": - locals[local.name] = local - for var in s.outputs: - if var.name in locals: - local = locals[var.name] - else: - local = Local.unused(var) - stack.push(local) - return stack + apply_stack_effect(stack, s) + result.append(stack.copy()) + return result + @dataclass class Storage: From 4fd9eb2aca21489ea0841155cdb81cb30dda73d3 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 26 Nov 2024 02:00:46 +0100 Subject: [PATCH 018/427] Fix typo: Use AsyncFor element access in codegen (#127278) Use AsyncFor element access in codegen --- Python/codegen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/codegen.c b/Python/codegen.c index bce3b94b27a45d..dbf36cdc0b7908 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -2033,7 +2033,7 @@ codegen_async_for(compiler *c, stmt_ty s) ADDOP(c, loc, END_ASYNC_FOR); /* `else` block */ - VISIT_SEQ(c, stmt, s->v.For.orelse); + VISIT_SEQ(c, stmt, s->v.AsyncFor.orelse); USE_LABEL(c, end); return SUCCESS; From 733fe59206e04141fd5cf65606ebd3d42a996226 Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Tue, 26 Nov 2024 17:17:54 +0900 Subject: [PATCH 019/427] gh-101100: Fix sphinx warnings in `howto/*` (#127084) --- Doc/howto/descriptor.rst | 100 +++++++++++++++++++------------------- Doc/howto/enum.rst | 101 ++++++++++++++++++++------------------- Doc/tools/.nitignore | 2 - 3 files changed, 101 insertions(+), 102 deletions(-) diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 01264bfe823746..f6c3e473f1c36d 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -42,7 +42,7 @@ add new capabilities one by one. Simple example: A descriptor that returns a constant ---------------------------------------------------- -The :class:`Ten` class is a descriptor whose :meth:`__get__` method always +The :class:`!Ten` class is a descriptor whose :meth:`~object.__get__` method always returns the constant ``10``: .. testcode:: @@ -120,10 +120,10 @@ different, updated answers each time:: 2 Besides showing how descriptors can run computations, this example also -reveals the purpose of the parameters to :meth:`__get__`. The *self* +reveals the purpose of the parameters to :meth:`~object.__get__`. The *self* parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that -lets the :meth:`__get__` method learn the target directory. The *objtype* +lets the :meth:`~object.__get__` method learn the target directory. The *objtype* parameter is the class *Directory*. @@ -133,7 +133,7 @@ Managed attributes A popular use for descriptors is managing access to instance data. The descriptor is assigned to a public attribute in the class dictionary while the actual data is stored as a private attribute in the instance dictionary. The -descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when +descriptor's :meth:`~object.__get__` and :meth:`~object.__set__` methods are triggered when the public attribute is accessed. In the following example, *age* is the public attribute and *_age* is the @@ -215,9 +215,9 @@ Customized names When a class uses descriptors, it can inform each descriptor about which variable name was used. -In this example, the :class:`Person` class has two descriptor instances, -*name* and *age*. When the :class:`Person` class is defined, it makes a -callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can +In this example, the :class:`!Person` class has two descriptor instances, +*name* and *age*. When the :class:`!Person` class is defined, it makes a +callback to :meth:`~object.__set_name__` in *LoggedAccess* so that the field names can be recorded, giving each descriptor its own *public_name* and *private_name*: .. testcode:: @@ -253,8 +253,8 @@ be recorded, giving each descriptor its own *public_name* and *private_name*: def birthday(self): self.age += 1 -An interactive session shows that the :class:`Person` class has called -:meth:`__set_name__` so that the field names would be recorded. Here +An interactive session shows that the :class:`!Person` class has called +:meth:`~object.__set_name__` so that the field names would be recorded. Here we call :func:`vars` to look up the descriptor without triggering it: .. doctest:: @@ -294,10 +294,10 @@ The two *Person* instances contain only the private names: Closing thoughts ---------------- -A :term:`descriptor` is what we call any object that defines :meth:`__get__`, -:meth:`__set__`, or :meth:`__delete__`. +A :term:`descriptor` is what we call any object that defines :meth:`~object.__get__`, +:meth:`~object.__set__`, or :meth:`~object.__delete__`. -Optionally, descriptors can have a :meth:`__set_name__` method. This is only +Optionally, descriptors can have a :meth:`~object.__set_name__` method. This is only used in cases where a descriptor needs to know either the class where it was created or the name of class variable it was assigned to. (This method, if present, is called even if the class is not a descriptor.) @@ -337,7 +337,7 @@ any data, it verifies that the new value meets various type and range restrictions. If those restrictions aren't met, it raises an exception to prevent data corruption at its source. -This :class:`Validator` class is both an :term:`abstract base class` and a +This :class:`!Validator` class is both an :term:`abstract base class` and a managed attribute descriptor: .. testcode:: @@ -360,8 +360,8 @@ managed attribute descriptor: def validate(self, value): pass -Custom validators need to inherit from :class:`Validator` and must supply a -:meth:`validate` method to test various restrictions as needed. +Custom validators need to inherit from :class:`!Validator` and must supply a +:meth:`!validate` method to test various restrictions as needed. Custom validators @@ -369,13 +369,13 @@ Custom validators Here are three practical data validation utilities: -1) :class:`OneOf` verifies that a value is one of a restricted set of options. +1) :class:`!OneOf` verifies that a value is one of a restricted set of options. -2) :class:`Number` verifies that a value is either an :class:`int` or +2) :class:`!Number` verifies that a value is either an :class:`int` or :class:`float`. Optionally, it verifies that a value is between a given minimum or maximum. -3) :class:`String` verifies that a value is a :class:`str`. Optionally, it +3) :class:`!String` verifies that a value is a :class:`str`. Optionally, it validates a given minimum or maximum length. It can validate a user-defined `predicate `_ as well. @@ -501,8 +501,8 @@ Definition and introduction --------------------------- In general, a descriptor is an attribute value that has one of the methods in -the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, -and :meth:`__delete__`. If any of those methods are defined for an +the descriptor protocol. Those methods are :meth:`~object.__get__`, :meth:`~object.__set__`, +and :meth:`~object.__delete__`. If any of those methods are defined for an attribute, it is said to be a :term:`descriptor`. The default behavior for attribute access is to get, set, or delete the @@ -534,8 +534,8 @@ That is all there is to it. Define any of these methods and an object is considered a descriptor and can override default behavior upon being looked up as an attribute. -If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered -a data descriptor. Descriptors that only define :meth:`__get__` are called +If an object defines :meth:`~object.__set__` or :meth:`~object.__delete__`, it is considered +a data descriptor. Descriptors that only define :meth:`~object.__get__` are called non-data descriptors (they are often used for methods but other uses are possible). @@ -545,9 +545,9 @@ has an entry with the same name as a data descriptor, the data descriptor takes precedence. If an instance's dictionary has an entry with the same name as a non-data descriptor, the dictionary entry takes precedence. -To make a read-only data descriptor, define both :meth:`__get__` and -:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when -called. Defining the :meth:`__set__` method with an exception raising +To make a read-only data descriptor, define both :meth:`~object.__get__` and +:meth:`~object.__set__` with the :meth:`~object.__set__` raising an :exc:`AttributeError` when +called. Defining the :meth:`~object.__set__` method with an exception raising placeholder is enough to make it a data descriptor. @@ -574,7 +574,7 @@ Invocation from an instance Instance lookup scans through a chain of namespaces giving data descriptors the highest priority, followed by instance variables, then non-data -descriptors, then class variables, and lastly :meth:`__getattr__` if it is +descriptors, then class variables, and lastly :meth:`~object.__getattr__` if it is provided. If a descriptor is found for ``a.x``, then it is invoked with: @@ -719,12 +719,12 @@ a pure Python equivalent: >>> object_getattribute(u2, 'x') == u2.x == (D1, u2, U2) True -Note, there is no :meth:`__getattr__` hook in the :meth:`__getattribute__` -code. That is why calling :meth:`__getattribute__` directly or with -``super().__getattribute__`` will bypass :meth:`__getattr__` entirely. +Note, there is no :meth:`~object.__getattr__` hook in the :meth:`~object.__getattribute__` +code. That is why calling :meth:`~object.__getattribute__` directly or with +``super().__getattribute__`` will bypass :meth:`~object.__getattr__` entirely. Instead, it is the dot operator and the :func:`getattr` function that are -responsible for invoking :meth:`__getattr__` whenever :meth:`__getattribute__` +responsible for invoking :meth:`~object.__getattr__` whenever :meth:`~object.__getattribute__` raises an :exc:`AttributeError`. Their logic is encapsulated in a helper function: @@ -776,8 +776,8 @@ Invocation from a class ----------------------- The logic for a dotted lookup such as ``A.x`` is in -:meth:`type.__getattribute__`. The steps are similar to those for -:meth:`object.__getattribute__` but the instance dictionary lookup is replaced +:meth:`!type.__getattribute__`. The steps are similar to those for +:meth:`!object.__getattribute__` but the instance dictionary lookup is replaced by a search through the class's :term:`method resolution order`. If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. @@ -789,7 +789,7 @@ The full C implementation can be found in :c:func:`!type_getattro` and Invocation from super --------------------- -The logic for super's dotted lookup is in the :meth:`__getattribute__` method for +The logic for super's dotted lookup is in the :meth:`~object.__getattribute__` method for object returned by :func:`super`. A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` @@ -806,21 +806,21 @@ The full C implementation can be found in :c:func:`!super_getattro` in Summary of invocation logic --------------------------- -The mechanism for descriptors is embedded in the :meth:`__getattribute__` +The mechanism for descriptors is embedded in the :meth:`~object.__getattribute__` methods for :class:`object`, :class:`type`, and :func:`super`. The important points to remember are: -* Descriptors are invoked by the :meth:`__getattribute__` method. +* Descriptors are invoked by the :meth:`~object.__getattribute__` method. * Classes inherit this machinery from :class:`object`, :class:`type`, or :func:`super`. -* Overriding :meth:`__getattribute__` prevents automatic descriptor calls +* Overriding :meth:`~object.__getattribute__` prevents automatic descriptor calls because all the descriptor logic is in that method. -* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make - different calls to :meth:`__get__`. The first includes the instance and may +* :meth:`!object.__getattribute__` and :meth:`!type.__getattribute__` make + different calls to :meth:`~object.__get__`. The first includes the instance and may include the class. The second puts in ``None`` for the instance and always includes the class. @@ -835,16 +835,16 @@ Automatic name notification Sometimes it is desirable for a descriptor to know what class variable name it was assigned to. When a new class is created, the :class:`type` metaclass scans the dictionary of the new class. If any of the entries are descriptors -and if they define :meth:`__set_name__`, that method is called with two +and if they define :meth:`~object.__set_name__`, that method is called with two arguments. The *owner* is the class where the descriptor is used, and the *name* is the class variable the descriptor was assigned to. The implementation details are in :c:func:`!type_new` and :c:func:`!set_names` in :source:`Objects/typeobject.c`. -Since the update logic is in :meth:`type.__new__`, notifications only take +Since the update logic is in :meth:`!type.__new__`, notifications only take place at the time of class creation. If descriptors are added to the class -afterwards, :meth:`__set_name__` will need to be called manually. +afterwards, :meth:`~object.__set_name__` will need to be called manually. ORM example @@ -873,7 +873,7 @@ care of lookups or updates: conn.execute(self.store, [value, obj.key]) conn.commit() -We can use the :class:`Field` class to define `models +We can use the :class:`!Field` class to define `models `_ that describe the schema for each table in a database: @@ -1140,7 +1140,7 @@ to wrap access to the value attribute in a property data descriptor: self.recalc() return self._value -Either the built-in :func:`property` or our :func:`Property` equivalent would +Either the built-in :func:`property` or our :func:`!Property` equivalent would work in this example. @@ -1187,7 +1187,7 @@ roughly equivalent to: return self To support automatic creation of methods, functions include the -:meth:`__get__` method for binding methods during attribute access. This +:meth:`~object.__get__` method for binding methods during attribute access. This means that functions are non-data descriptors that return bound methods during dotted lookup from an instance. Here's how it works: @@ -1231,19 +1231,19 @@ The function has a :term:`qualified name` attribute to support introspection: 'D.f' Accessing the function through the class dictionary does not invoke -:meth:`__get__`. Instead, it just returns the underlying function object:: +:meth:`~object.__get__`. Instead, it just returns the underlying function object:: >>> D.__dict__['f'] -Dotted access from a class calls :meth:`__get__` which just returns the +Dotted access from a class calls :meth:`~object.__get__` which just returns the underlying function unchanged:: >>> D.f The interesting behavior occurs during dotted access from an instance. The -dotted lookup calls :meth:`__get__` which returns a bound method object:: +dotted lookup calls :meth:`~object.__get__` which returns a bound method object:: >>> d = D() >>> d.f @@ -1268,7 +1268,7 @@ Kinds of methods Non-data descriptors provide a simple mechanism for variations on the usual patterns of binding functions into methods. -To recap, functions have a :meth:`__get__` method so that they can be converted +To recap, functions have a :meth:`~object.__get__` method so that they can be converted to a method when accessed as attributes. The non-data descriptor transforms an ``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` becomes ``f(*args)``. @@ -1671,7 +1671,7 @@ by member descriptors: 'Emulate member_repr() in Objects/descrobject.c' return f'' -The :meth:`type.__new__` method takes care of adding member objects to class +The :meth:`!type.__new__` method takes care of adding member objects to class variables: .. testcode:: @@ -1722,7 +1722,7 @@ Python: ) super().__delattr__(name) -To use the simulation in a real class, just inherit from :class:`Object` and +To use the simulation in a real class, just inherit from :class:`!Object` and set the :term:`metaclass` to :class:`Type`: .. testcode:: diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 66929b4104d8de..6441b7aed1eda8 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -64,12 +64,12 @@ The *type* of an enumeration member is the enum it belongs to:: >>> isinstance(Weekday.FRIDAY, Weekday) True -Enum members have an attribute that contains just their :attr:`name`:: +Enum members have an attribute that contains just their :attr:`!name`:: >>> print(Weekday.TUESDAY.name) TUESDAY -Likewise, they have an attribute for their :attr:`value`:: +Likewise, they have an attribute for their :attr:`!value`:: >>> Weekday.WEDNESDAY.value @@ -77,17 +77,18 @@ Likewise, they have an attribute for their :attr:`value`:: Unlike many languages that treat enumerations solely as name/value pairs, Python Enums can have behavior added. For example, :class:`datetime.date` -has two methods for returning the weekday: :meth:`weekday` and :meth:`isoweekday`. +has two methods for returning the weekday: +:meth:`~datetime.date.weekday` and :meth:`~datetime.date.isoweekday`. The difference is that one of them counts from 0-6 and the other from 1-7. -Rather than keep track of that ourselves we can add a method to the :class:`Weekday` -enum to extract the day from the :class:`date` instance and return the matching +Rather than keep track of that ourselves we can add a method to the :class:`!Weekday` +enum to extract the day from the :class:`~datetime.date` instance and return the matching enum member:: @classmethod def from_date(cls, date): return cls(date.isoweekday()) -The complete :class:`Weekday` enum now looks like this:: +The complete :class:`!Weekday` enum now looks like this:: >>> class Weekday(Enum): ... MONDAY = 1 @@ -110,7 +111,7 @@ Now we can find out what today is! Observe:: Of course, if you're reading this on some other day, you'll see that day instead. -This :class:`Weekday` enum is great if our variable only needs one day, but +This :class:`!Weekday` enum is great if our variable only needs one day, but what if we need several? Maybe we're writing a function to plot chores during a week, and don't want to use a :class:`list` -- we could use a different type of :class:`Enum`:: @@ -128,7 +129,7 @@ of :class:`Enum`:: We've changed two things: we're inherited from :class:`Flag`, and the values are all powers of 2. -Just like the original :class:`Weekday` enum above, we can have a single selection:: +Just like the original :class:`!Weekday` enum above, we can have a single selection:: >>> first_week_day = Weekday.MONDAY >>> first_week_day @@ -203,7 +204,7 @@ If you want to access enum members by *name*, use item access:: >>> Color['GREEN'] -If you have an enum member and need its :attr:`name` or :attr:`value`:: +If you have an enum member and need its :attr:`!name` or :attr:`!value`:: >>> member = Color.RED >>> member.name @@ -284,7 +285,7 @@ If the exact value is unimportant you can use :class:`auto`:: >>> [member.value for member in Color] [1, 2, 3] -The values are chosen by :func:`_generate_next_value_`, which can be +The values are chosen by :func:`~Enum._generate_next_value_`, which can be overridden:: >>> class AutoName(Enum): @@ -303,7 +304,7 @@ overridden:: .. note:: - The :meth:`_generate_next_value_` method must be defined before any members. + The :meth:`~Enum._generate_next_value_` method must be defined before any members. Iteration --------- @@ -424,18 +425,18 @@ Then:: The rules for what is allowed are as follows: names that start and end with a single underscore are reserved by enum and cannot be used; all other attributes defined within an enumeration will become members of this -enumeration, with the exception of special methods (:meth:`__str__`, -:meth:`__add__`, etc.), descriptors (methods are also descriptors), and -variable names listed in :attr:`_ignore_`. +enumeration, with the exception of special methods (:meth:`~object.__str__`, +:meth:`~object.__add__`, etc.), descriptors (methods are also descriptors), and +variable names listed in :attr:`~Enum._ignore_`. -Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__`, +Note: if your enumeration defines :meth:`~object.__new__` and/or :meth:`~object.__init__`, any value(s) given to the enum member will be passed into those methods. See `Planet`_ for an example. .. note:: - The :meth:`__new__` method, if defined, is used during creation of the Enum - members; it is then replaced by Enum's :meth:`__new__` which is used after + The :meth:`~object.__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's :meth:`~object.__new__` which is used after class creation for lookup of existing members. See :ref:`new-vs-init` for more details. @@ -544,7 +545,7 @@ from that module. nested in other classes. It is possible to modify how enum members are pickled/unpickled by defining -:meth:`__reduce_ex__` in the enumeration class. The default method is by-value, +:meth:`~object.__reduce_ex__` in the enumeration class. The default method is by-value, but enums with complicated values may want to use by-name:: >>> import enum @@ -580,7 +581,7 @@ values. The last two options enable assigning arbitrary values to enumerations; the others auto-assign increasing integers starting with 1 (use the ``start`` parameter to specify a different starting value). A new class derived from :class:`Enum` is returned. In other words, the above -assignment to :class:`Animal` is equivalent to:: +assignment to :class:`!Animal` is equivalent to:: >>> class Animal(Enum): ... ANT = 1 @@ -891,7 +892,7 @@ simple to implement independently:: pass This demonstrates how similar derived enumerations can be defined; for example -a :class:`FloatEnum` that mixes in :class:`float` instead of :class:`int`. +a :class:`!FloatEnum` that mixes in :class:`float` instead of :class:`int`. Some rules: @@ -905,32 +906,32 @@ Some rules: additional type, all the members must have values of that type, e.g. :class:`int` above. This restriction does not apply to mix-ins which only add methods and don't specify another type. -4. When another data type is mixed in, the :attr:`value` attribute is *not the +4. When another data type is mixed in, the :attr:`~Enum.value` attribute is *not the same* as the enum member itself, although it is equivalent and will compare equal. -5. A ``data type`` is a mixin that defines :meth:`__new__`, or a +5. A ``data type`` is a mixin that defines :meth:`~object.__new__`, or a :class:`~dataclasses.dataclass` 6. %-style formatting: ``%s`` and ``%r`` call the :class:`Enum` class's - :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as + :meth:`~object.__str__` and :meth:`~object.__repr__` respectively; other codes (such as ``%i`` or ``%h`` for IntEnum) treat the enum member as its mixed-in type. 7. :ref:`Formatted string literals `, :meth:`str.format`, - and :func:`format` will use the enum's :meth:`__str__` method. + and :func:`format` will use the enum's :meth:`~object.__str__` method. .. note:: Because :class:`IntEnum`, :class:`IntFlag`, and :class:`StrEnum` are designed to be drop-in replacements for existing constants, their - :meth:`__str__` method has been reset to their data types' - :meth:`__str__` method. + :meth:`~object.__str__` method has been reset to their data types' + :meth:`~object.__str__` method. .. _new-vs-init: -When to use :meth:`__new__` vs. :meth:`__init__` ------------------------------------------------- +When to use :meth:`~object.__new__` vs. :meth:`~object.__init__` +---------------------------------------------------------------- -:meth:`__new__` must be used whenever you want to customize the actual value of +:meth:`~object.__new__` must be used whenever you want to customize the actual value of the :class:`Enum` member. Any other modifications may go in either -:meth:`__new__` or :meth:`__init__`, with :meth:`__init__` being preferred. +:meth:`~object.__new__` or :meth:`~object.__init__`, with :meth:`~object.__init__` being preferred. For example, if you want to pass several items to the constructor, but only want one of them to be the value:: @@ -969,11 +970,11 @@ Finer Points Supported ``__dunder__`` names """""""""""""""""""""""""""""" -:attr:`__members__` is a read-only ordered mapping of ``member_name``:``member`` +:attr:`~enum.EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` items. It is only available on the class. -:meth:`__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`_value_` appropriately. Once +:meth:`~object.__new__`, if specified, must create and return the enum members; it is +also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once all the members are created it is no longer used. @@ -989,9 +990,9 @@ Supported ``_sunder_`` names from the final class - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden -- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing +- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing member. -- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an +- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an existing member. See `MultiValueEnum`_ for an example. .. note:: @@ -1009,7 +1010,7 @@ Supported ``_sunder_`` names .. versionadded:: 3.7 ``_ignore_`` .. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` -To help keep Python 2 / Python 3 code in sync an :attr:`_order_` attribute can +To help keep Python 2 / Python 3 code in sync an :attr:`~Enum._order_` attribute can be provided. It will be checked against the actual order of the enumeration and raise an error if the two do not match:: @@ -1027,7 +1028,7 @@ and raise an error if the two do not match:: .. note:: - In Python 2 code the :attr:`_order_` attribute is necessary as definition + In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition order is lost before it can be recorded. @@ -1216,12 +1217,12 @@ Enum Classes ^^^^^^^^^^^^ The :class:`EnumType` metaclass is responsible for providing the -:meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that +:meth:`~object.__contains__`, :meth:`~object.__dir__`, :meth:`~object.__iter__` and other methods that allow one to do things with an :class:`Enum` class that fail on a typical class, such as ``list(Color)`` or ``some_enum_var in Color``. :class:`EnumType` is responsible for ensuring that various other methods on the final :class:`Enum` -class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`, -:meth:`__str__` and :meth:`__repr__`). +class are correct (such as :meth:`~object.__new__`, :meth:`~object.__getnewargs__`, +:meth:`~object.__str__` and :meth:`~object.__repr__`). Flag Classes ^^^^^^^^^^^^ @@ -1236,7 +1237,7 @@ Enum Members (aka instances) The most interesting thing about enum members is that they are singletons. :class:`EnumType` creates them all while it is creating the enum class itself, -and then puts a custom :meth:`__new__` in place to ensure that no new ones are +and then puts a custom :meth:`~object.__new__` in place to ensure that no new ones are ever instantiated by returning only the existing member instances. Flag Members @@ -1284,7 +1285,7 @@ is. There are several ways to define this type of simple enumeration: - use instances of :class:`auto` for the value - use instances of :class:`object` as the value - use a descriptive string as the value -- use a tuple as the value and a custom :meth:`__new__` to replace the +- use a tuple as the value and a custom :meth:`~object.__new__` to replace the tuple with an :class:`int` value Using any of these methods signifies to the user that these values are not @@ -1320,7 +1321,7 @@ Using :class:`object` would look like:: > This is also a good example of why you might want to write your own -:meth:`__repr__`:: +:meth:`~object.__repr__`:: >>> class Color(Enum): ... RED = object() @@ -1348,10 +1349,10 @@ Using a string as the value would look like:: -Using a custom :meth:`__new__` -"""""""""""""""""""""""""""""" +Using a custom :meth:`~object.__new__` +"""""""""""""""""""""""""""""""""""""" -Using an auto-numbering :meth:`__new__` would look like:: +Using an auto-numbering :meth:`~object.__new__` would look like:: >>> class AutoNumber(Enum): ... def __new__(cls): @@ -1397,8 +1398,8 @@ to handle any extra arguments:: .. note:: - The :meth:`__new__` method, if defined, is used during creation of the Enum - members; it is then replaced by Enum's :meth:`__new__` which is used after + The :meth:`~object.__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's :meth:`~object.__new__` which is used after class creation for lookup of existing members. .. warning:: @@ -1504,7 +1505,7 @@ Supports having more than one value per member:: Planet ^^^^^^ -If :meth:`__new__` or :meth:`__init__` is defined, the value of the enum member +If :meth:`~object.__new__` or :meth:`~object.__init__` is defined, the value of the enum member will be passed to those methods:: >>> class Planet(Enum): @@ -1535,7 +1536,7 @@ will be passed to those methods:: TimePeriod ^^^^^^^^^^ -An example to show the :attr:`_ignore_` attribute in use:: +An example to show the :attr:`~Enum._ignore_` attribute in use:: >>> from datetime import timedelta >>> class Period(timedelta, Enum): diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 66914f79f3d4ec..711c0b64095bd2 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -13,8 +13,6 @@ Doc/c-api/type.rst Doc/c-api/typeobj.rst Doc/extending/extending.rst Doc/glossary.rst -Doc/howto/descriptor.rst -Doc/howto/enum.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/asyncio-subprocess.rst From f46d8475749a0eadbc1f37079906a8e1ed5d56dc Mon Sep 17 00:00:00 2001 From: Beomsoo Kim Date: Tue, 26 Nov 2024 17:54:02 +0900 Subject: [PATCH 020/427] gh-126946: Improve error message in getopt.do_longs based on existing comment (GH-126871) Include a list of possibilities for not unique prefix. --- Lib/getopt.py | 10 ++++++---- Lib/test/translationdata/getopt/msgids.txt | 2 +- .../2024-11-18-16-43-11.gh-issue-126946.52Ou-B.rst | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-18-16-43-11.gh-issue-126946.52Ou-B.rst diff --git a/Lib/getopt.py b/Lib/getopt.py index a9c452a601ee81..25f3e2439b3e35 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -185,11 +185,13 @@ def long_has_args(opt, longopts): return True, opt elif opt + '=?' in possibilities: return '?', opt - # No exact match, so better be unique. + # Possibilities must be unique to be accepted if len(possibilities) > 1: - # XXX since possibilities contains all valid continuations, might be - # nice to work them into the error msg - raise GetoptError(_('option --%s not a unique prefix') % opt, opt) + raise GetoptError( + _("option --%s not a unique prefix; possible options: %s") + % (opt, ", ".join(possibilities)), + opt, + ) assert len(possibilities) == 1 unique_match = possibilities[0] if unique_match.endswith('=?'): diff --git a/Lib/test/translationdata/getopt/msgids.txt b/Lib/test/translationdata/getopt/msgids.txt index 1ffab1f31abad5..5c0c02d452d156 100644 --- a/Lib/test/translationdata/getopt/msgids.txt +++ b/Lib/test/translationdata/getopt/msgids.txt @@ -1,6 +1,6 @@ option -%s not recognized option -%s requires argument option --%s must not have an argument -option --%s not a unique prefix +option --%s not a unique prefix; possible options: %s option --%s not recognized option --%s requires argument \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2024-11-18-16-43-11.gh-issue-126946.52Ou-B.rst b/Misc/NEWS.d/next/Library/2024-11-18-16-43-11.gh-issue-126946.52Ou-B.rst new file mode 100644 index 00000000000000..448055ccfdff40 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-18-16-43-11.gh-issue-126946.52Ou-B.rst @@ -0,0 +1,3 @@ +Improve the :exc:`~getopt.GetoptError` error message when a long option +prefix matches multiple accepted options in :func:`getopt.getopt` and +:func:`getopt.gnu_getopt`. From 3c7a90a83146dc6e55f6f9ecd9af0bf9682f98a6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Nov 2024 10:21:57 +0100 Subject: [PATCH 021/427] gh-122273: Support PyREPL history on Windows (#127141) Co-authored-by: devdanzin <74280297+devdanzin@users.noreply.github.com> --- Lib/_pyrepl/readline.py | 4 +- Lib/site.py | 55 ++++++++++++------- ...-11-22-09-23-41.gh-issue-122273.H8M6fd.rst | 1 + 3 files changed, 40 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-22-09-23-41.gh-issue-122273.H8M6fd.rst diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 5e1d3085874380..888185eb03be66 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -450,7 +450,9 @@ def read_history_file(self, filename: str = gethistoryfile()) -> None: def write_history_file(self, filename: str = gethistoryfile()) -> None: maxlength = self.saved_history_length history = self.get_reader().get_trimmed_history(maxlength) - with open(os.path.expanduser(filename), "w", encoding="utf-8") as f: + f = open(os.path.expanduser(filename), "w", + encoding="utf-8", newline="\n") + with f: for entry in history: entry = entry.replace("\n", "\r\n") # multiline history support f.write(entry + "\n") diff --git a/Lib/site.py b/Lib/site.py index 54f07ab5b4e6d5..abf4b520244554 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -498,9 +498,18 @@ def register_readline(): PYTHON_BASIC_REPL = False import atexit + + try: + try: + import readline + except ImportError: + readline = None + else: + import rlcompleter # noqa: F401 + except ImportError: + return + try: - import readline - import rlcompleter # noqa: F401 if PYTHON_BASIC_REPL: CAN_USE_PYREPL = False else: @@ -508,30 +517,36 @@ def register_readline(): sys.path = [p for p in original_path if p != ''] try: import _pyrepl.readline - import _pyrepl.unix_console + if os.name == "nt": + import _pyrepl.windows_console + console_errors = (_pyrepl.windows_console._error,) + else: + import _pyrepl.unix_console + console_errors = _pyrepl.unix_console._error from _pyrepl.main import CAN_USE_PYREPL finally: sys.path = original_path except ImportError: return - # Reading the initialization (config) file may not be enough to set a - # completion key, so we set one first and then read the file. - if readline.backend == 'editline': - readline.parse_and_bind('bind ^I rl_complete') - else: - readline.parse_and_bind('tab: complete') + if readline is not None: + # Reading the initialization (config) file may not be enough to set a + # completion key, so we set one first and then read the file. + if readline.backend == 'editline': + readline.parse_and_bind('bind ^I rl_complete') + else: + readline.parse_and_bind('tab: complete') - try: - readline.read_init_file() - except OSError: - # An OSError here could have many causes, but the most likely one - # is that there's no .inputrc file (or .editrc file in the case of - # Mac OS X + libedit) in the expected location. In that case, we - # want to ignore the exception. - pass + try: + readline.read_init_file() + except OSError: + # An OSError here could have many causes, but the most likely one + # is that there's no .inputrc file (or .editrc file in the case of + # Mac OS X + libedit) in the expected location. In that case, we + # want to ignore the exception. + pass - if readline.get_current_history_length() == 0: + if readline is None or readline.get_current_history_length() == 0: # If no history was loaded, default to .python_history, # or PYTHON_HISTORY. # The guard is necessary to avoid doubling history size at @@ -542,8 +557,10 @@ def register_readline(): if CAN_USE_PYREPL: readline_module = _pyrepl.readline - exceptions = (OSError, *_pyrepl.unix_console._error) + exceptions = (OSError, *console_errors) else: + if readline is None: + return readline_module = readline exceptions = OSError diff --git a/Misc/NEWS.d/next/Library/2024-11-22-09-23-41.gh-issue-122273.H8M6fd.rst b/Misc/NEWS.d/next/Library/2024-11-22-09-23-41.gh-issue-122273.H8M6fd.rst new file mode 100644 index 00000000000000..99071e05377e33 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-22-09-23-41.gh-issue-122273.H8M6fd.rst @@ -0,0 +1 @@ +Support PyREPL history on Windows. Patch by devdanzin and Victor Stinner. From 4ca2c82862e3810a7d68799df2bb5198a9afb219 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 26 Nov 2024 09:46:46 +0000 Subject: [PATCH 022/427] gh-124873: Skip timerfd tests on Android (#127279) * Revert "[3.13] gh-124873: Tolerate 100 ms in TimerfdTests on Android (GH-127101) (#127105)" This reverts commit c09366b1fed2289530581505834b2b262120a7c7. * Skip timerfd tests on Android. Co-authored-by: Victor Stinner --- Lib/test/test_os.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 99515dfc71f9ba..f3d2ceb263f6f4 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4174,12 +4174,13 @@ def test_eventfd_select(self): os.eventfd_read(fd) @unittest.skipUnless(hasattr(os, 'timerfd_create'), 'requires os.timerfd_create') +@unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): # 1 ms accuracy is reliably achievable on every platform except Android - # emulators, where we allow 100 ms (gh-124873). + # emulators, where we allow 10 ms (gh-108277). if sys.platform == "android" and platform.android_ver().is_emulator: - CLOCK_RES_PLACES = 1 + CLOCK_RES_PLACES = 2 else: CLOCK_RES_PLACES = 3 From 0c1feebf95440e00ffab14ada24c139eaabde628 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 26 Nov 2024 11:06:57 +0100 Subject: [PATCH 023/427] gh-113993: InternalDocs: Add String Interning to README (#127250) --- InternalDocs/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/InternalDocs/README.md b/InternalDocs/README.md index dbc858b276833c..f6aa3db3b384af 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -30,6 +30,8 @@ Runtime Objects - [Frames](frames.md) +- [String Interning](string_interning.md) + Program Execution --- From ab237ff81d2201c70aaacf51f0c033df334e5d07 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Tue, 26 Nov 2024 08:05:07 -0500 Subject: [PATCH 024/427] Doc: Typo fix: nrace -> race (#127288) --- Doc/library/concurrent.futures.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index a57120e88103de..5a950081a1c98d 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -252,7 +252,7 @@ This results in several benefits that help balance the extra effort, including true multi-core parallelism, For example, code written this way can make it easier to reason about concurrency. Another major benefit is that you don't have to deal with several of the -big pain points of using threads, like nrace conditions. +big pain points of using threads, like race conditions. Each worker's interpreter is isolated from all the other interpreters. "Isolated" means each interpreter has its own runtime state and From 2b0e2b2893a821ca36cd65a204bed932741ac189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Tue, 26 Nov 2024 13:46:33 +0000 Subject: [PATCH 025/427] GH-126985: move pyvenv.cfg detection from site to getpath (#126987) --- Doc/c-api/init_config.rst | 13 ++++ Doc/library/site.rst | 24 ++++--- Doc/library/sys.rst | 64 +++++++++++------- Doc/library/sys_path_init.rst | 49 +++++++++++--- Doc/library/venv.rst | 3 + Lib/site.py | 11 ++- Lib/sysconfig/__init__.py | 18 ++--- Lib/test/test_embed.py | 4 +- Lib/test/test_getpath.py | 32 ++++----- Lib/test/test_sysconfig.py | 67 +------------------ ...-11-18-23-42-06.gh-issue-126985.7XplY9.rst | 3 + Modules/getpath.py | 43 +++++++----- 12 files changed, 176 insertions(+), 155 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index e621545cffcee3..d6569ddcf586fa 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1590,9 +1590,22 @@ If a ``._pth`` file is present: * Set :c:member:`~PyConfig.site_import` to ``0``. * Set :c:member:`~PyConfig.safe_path` to ``1``. +If :c:member:`~PyConfig.home` is not set and a ``pyvenv.cfg`` file is present in +the same directory as :c:member:`~PyConfig.executable`, or its parent, +:c:member:`~PyConfig.prefix` and :c:member:`~PyConfig.exec_prefix` are set that +location. When this happens, :c:member:`~PyConfig.base_prefix` and +:c:member:`~PyConfig.base_exec_prefix` still keep their value, pointing to the +base installation. See :ref:`sys-path-init-virtual-environments` for more +information. + The ``__PYVENV_LAUNCHER__`` environment variable is used to set :c:member:`PyConfig.base_executable`. +.. versionchanged:: 3.14 + + :c:member:`~PyConfig.prefix`, and :c:member:`~PyConfig.exec_prefix`, are now + set to the ``pyvenv.cfg`` directory. This was previously done by :mod:`site`, + therefore affected by :option:`-S`. .. _pyinitconfig_api: diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 4508091f679dc7..5f2a0f610e1aa5 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -49,14 +49,22 @@ added path for configuration files. identified by the "t" suffix in the version-specific directory name, such as :file:`lib/python3.13t/`. -If a file named "pyvenv.cfg" exists one directory above sys.executable, -sys.prefix and sys.exec_prefix are set to that directory and -it is also checked for site-packages (sys.base_prefix and -sys.base_exec_prefix will always be the "real" prefixes of the Python -installation). If "pyvenv.cfg" (a bootstrap configuration file) contains -the key "include-system-site-packages" set to anything other than "true" -(case-insensitive), the system-level prefixes will not be -searched for site-packages; otherwise they will. +.. versionchanged:: 3.14 + + :mod:`site` is no longer responsible for updating :data:`sys.prefix` and + :data:`sys.exec_prefix` on :ref:`sys-path-init-virtual-environments`. This is + now done during the :ref:`path initialization `. As a result, + under :ref:`sys-path-init-virtual-environments`, :data:`sys.prefix` and + :data:`sys.exec_prefix` no longer depend on the :mod:`site` initialization, + and are therefore unaffected by :option:`-S`. + +.. _site-virtual-environments-configuration: + +When running under a :ref:`virtual environment `, +the ``pyvenv.cfg`` file in :data:`sys.prefix` is checked for site-specific +configurations. If the ``include-system-site-packages`` key exists and is set to +``true`` (case-insensitive), the system-level prefixes will be searched for +site-packages, otherwise they won't. .. index:: single: # (hash); comment diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 773e5d4185dfaf..dd6293c722e7ad 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -130,27 +130,26 @@ always available. .. data:: base_exec_prefix - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`exec_prefix`. If not running in a - :ref:`virtual environment `, the values will stay the same; if - ``site.py`` finds that a virtual environment is in use, the values of - :data:`prefix` and :data:`exec_prefix` will be changed to point to the - virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). + Equivalent to :data:`exec_prefix`, but refering to the base Python installation. + + When running under :ref:`sys-path-init-virtual-environments`, + :data:`exec_prefix` gets overwritten to the virtual environment prefix. + :data:`base_exec_prefix`, conversely, does not change, and always points to + the base Python installation. + Refer to :ref:`sys-path-init-virtual-environments` for more information. .. versionadded:: 3.3 .. data:: base_prefix - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`prefix`. If not running in a :ref:`virtual environment `, the values - will stay the same; if ``site.py`` finds that a virtual environment is in - use, the values of :data:`prefix` and :data:`exec_prefix` will be changed to - point to the virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). + Equivalent to :data:`prefix`, but refering to the base Python installation. + + When running under :ref:`virtual environment `, + :data:`prefix` gets overwritten to the virtual environment prefix. + :data:`base_prefix`, conversely, does not change, and always points to + the base Python installation. + Refer to :ref:`sys-path-init-virtual-environments` for more information. .. versionadded:: 3.3 @@ -483,11 +482,19 @@ always available. .. note:: - If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual environment. - The value for the Python installation will still be available, via - :data:`base_exec_prefix`. + If a :ref:`virtual environment ` is in effect, this :data:`exec_prefix` + will point to the virtual environment. The value for the Python installation + will still be available, via :data:`base_exec_prefix`. + Refer to :ref:`sys-path-init-virtual-environments` for more information. + .. versionchanged:: 3.14 + + When running under a :ref:`virtual environment `, + :data:`prefix` and :data:`exec_prefix` are now set to the virtual + environment prefix by the :ref:`path initialization `, + instead of :mod:`site`. This means that :data:`prefix` and + :data:`exec_prefix` always point to the virtual environment, even when + :mod:`site` is disabled (:option:`-S`). .. data:: executable @@ -1483,10 +1490,21 @@ always available. argument to the :program:`configure` script. See :ref:`installation_paths` for derived paths. - .. note:: If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual - environment. The value for the Python installation will still be - available, via :data:`base_prefix`. + .. note:: + + If a :ref:`virtual environment ` is in effect, this :data:`prefix` + will point to the virtual environment. The value for the Python installation + will still be available, via :data:`base_prefix`. + Refer to :ref:`sys-path-init-virtual-environments` for more information. + + .. versionchanged:: 3.14 + + When running under a :ref:`virtual environment `, + :data:`prefix` and :data:`exec_prefix` are now set to the virtual + environment prefix by the :ref:`path initialization `, + instead of :mod:`site`. This means that :data:`prefix` and + :data:`exec_prefix` always point to the virtual environment, even when + :mod:`site` is disabled (:option:`-S`). .. data:: ps1 diff --git a/Doc/library/sys_path_init.rst b/Doc/library/sys_path_init.rst index a87a41cf829fa8..18fe32d9c7f10a 100644 --- a/Doc/library/sys_path_init.rst +++ b/Doc/library/sys_path_init.rst @@ -47,8 +47,15 @@ however on other platforms :file:`lib/python{majorversion}.{minorversion}/lib-dy ``exec_prefix``. On some platforms :file:`lib` may be :file:`lib64` or another value, see :data:`sys.platlibdir` and :envvar:`PYTHONPLATLIBDIR`. -Once found, ``prefix`` and ``exec_prefix`` are available at :data:`sys.prefix` and -:data:`sys.exec_prefix` respectively. +Once found, ``prefix`` and ``exec_prefix`` are available at +:data:`sys.base_prefix` and :data:`sys.base_exec_prefix` respectively. + +If :envvar:`PYTHONHOME` is not set, and a ``pyvenv.cfg`` file is found alongside +the main executable, or in its parent directory, :data:`sys.prefix` and +:data:`sys.exec_prefix` get set to the directory containing ``pyvenv.cfg``, +otherwise they are set to the same value as :data:`sys.base_prefix` and +:data:`sys.base_exec_prefix`, respectively. +This is used by :ref:`sys-path-init-virtual-environments`. Finally, the :mod:`site` module is processed and :file:`site-packages` directories are added to the module search path. A common way to customize the search path is @@ -60,18 +67,40 @@ the :mod:`site` module documentation. Certain command line options may further affect path calculations. See :option:`-E`, :option:`-I`, :option:`-s` and :option:`-S` for further details. -Virtual environments +.. versionchanged:: 3.14 + + :data:`sys.prefix` and :data:`sys.exec_prefix` are now set to the + ``pyvenv.cfg`` directory during the path initialization. This was previously + done by :mod:`site`, therefore affected by :option:`-S`. + +.. _sys-path-init-virtual-environments: + +Virtual Environments -------------------- -If Python is run in a virtual environment (as described at :ref:`tut-venv`) -then ``prefix`` and ``exec_prefix`` are specific to the virtual environment. +Virtual environments place a ``pyvenv.cfg`` file in their prefix, which causes +:data:`sys.prefix` and :data:`sys.exec_prefix` to point to them, instead of the +base installation. + +The ``prefix`` and ``exec_prefix`` values of the base installation are available +at :data:`sys.base_prefix` and :data:`sys.base_exec_prefix`. -If a ``pyvenv.cfg`` file is found alongside the main executable, or in the -directory one level above the executable, the following variations apply: +As well as being used as a marker to identify virtual environments, +``pyvenv.cfg`` may also be used to configure the :mod:`site` initialization. +Please refer to :mod:`site`'s +:ref:`virtual environments documentation `. + +.. note:: + + :envvar:`PYTHONHOME` overrides the ``pyvenv.cfg`` detection. + +.. note:: -* If ``home`` is an absolute path and :envvar:`PYTHONHOME` is not set, this - path is used instead of the path to the main executable when deducing ``prefix`` - and ``exec_prefix``. + There are other ways how "virtual environments" could be implemented, this + documentation referes implementations based on the ``pyvenv.cfg`` mechanism, + such as :mod:`venv`. Most virtual environment implementations follow the + model set by :mod:`venv`, but there may be exotic implementations that + diverge from it. _pth files ---------- diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 5205c6c211d9bf..bed799aedfdfb1 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -25,6 +25,9 @@ A virtual environment is created on top of an existing Python installation, known as the virtual environment's "base" Python, and may optionally be isolated from the packages in the base environment, so only those explicitly installed in the virtual environment are available. +See :ref:`sys-path-init-virtual-environments` and :mod:`site`'s +:ref:`virtual environments documentation ` +for more information. When used from within a virtual environment, common installation tools such as :pypi:`pip` will install Python packages into a virtual environment diff --git a/Lib/site.py b/Lib/site.py index abf4b520244554..92bd1ccdadd924 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -94,6 +94,12 @@ def _trace(message): print(message, file=sys.stderr) +def _warn(*args, **kwargs): + import warnings + + warnings.warn(*args, **kwargs) + + def makepath(*paths): dir = os.path.join(*paths) try: @@ -619,7 +625,10 @@ def venv(known_paths): elif key == 'home': sys._home = value - sys.prefix = sys.exec_prefix = site_prefix + if sys.prefix != site_prefix: + _warn(f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', RuntimeWarning) + if sys.exec_prefix != site_prefix: + _warn(f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', RuntimeWarning) # Doing this here ensures venv takes precedence over user-site addsitepackages(known_paths, [sys.prefix]) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 67a071963d8c7d..ee52700b51fd07 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -173,7 +173,9 @@ def joinuser(*args): _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' +_PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) +_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) # Mutex guarding initialization of _CONFIG_VARS. _CONFIG_VARS_LOCK = threading.RLock() @@ -465,10 +467,8 @@ def _init_config_vars(): # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _PREFIX = os.path.normpath(sys.prefix) - _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) - _CONFIG_VARS['prefix'] = _PREFIX # FIXME: This gets overwriten by _init_posix. - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['prefix'] = _PREFIX + _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT @@ -541,7 +541,6 @@ def get_config_vars(*args): With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ - global _CONFIG_VARS_INITIALIZED # Avoid claiming the lock once initialization is complete. if not _CONFIG_VARS_INITIALIZED: @@ -552,15 +551,6 @@ def get_config_vars(*args): # don't re-enter init_config_vars(). if _CONFIG_VARS is None: _init_config_vars() - else: - # If the site module initialization happened after _CONFIG_VARS was - # initialized, a virtual environment might have been activated, resulting in - # variables like sys.prefix changing their value, so we need to re-init the - # config vars (see GH-126789). - if _CONFIG_VARS['base'] != os.path.normpath(sys.prefix): - with _CONFIG_VARS_LOCK: - _CONFIG_VARS_INITIALIZED = False - _init_config_vars() if args: vals = [] diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index bf861ef06ee2d3..5c38b28322deb4 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1649,14 +1649,14 @@ def test_init_pyvenv_cfg(self): config = { 'base_prefix': sysconfig.get_config_var("prefix"), 'base_exec_prefix': exec_prefix, - 'exec_prefix': exec_prefix, + 'exec_prefix': tmpdir, + 'prefix': tmpdir, 'base_executable': base_executable, 'executable': executable, 'module_search_paths': paths, } if MS_WINDOWS: config['base_prefix'] = pyvenv_home - config['prefix'] = pyvenv_home config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib') config['use_frozen_modules'] = bool(not support.Py_DEBUG) else: diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index d5dcdad9614ecc..7e5c4a3d14ddc5 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -92,8 +92,8 @@ def test_venv_win32(self): ]) expected = dict( executable=r"C:\venv\Scripts\python.exe", - prefix=r"C:\Python", - exec_prefix=r"C:\Python", + prefix=r"C:\venv", + exec_prefix=r"C:\venv", base_executable=r"C:\Python\python.exe", base_prefix=r"C:\Python", base_exec_prefix=r"C:\Python", @@ -339,8 +339,8 @@ def test_venv_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/venv", + exec_prefix="/venv", base_executable="/usr/bin/python", base_prefix="/usr", base_exec_prefix="/usr", @@ -371,8 +371,8 @@ def test_venv_changed_name_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/venv", + exec_prefix="/venv", base_executable="/usr/bin/python3", base_prefix="/usr", base_exec_prefix="/usr", @@ -404,8 +404,8 @@ def test_venv_non_installed_zip_path_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/path/to/non-installed", - exec_prefix="/path/to/non-installed", + prefix="/venv", + exec_prefix="/venv", base_executable="/path/to/non-installed/bin/python", base_prefix="/path/to/non-installed", base_exec_prefix="/path/to/non-installed", @@ -435,8 +435,8 @@ def test_venv_changed_name_copy_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/venv", + exec_prefix="/venv", base_executable="/usr/bin/python9", base_prefix="/usr", base_exec_prefix="/usr", @@ -652,8 +652,8 @@ def test_venv_framework_macos(self): ]) expected = dict( executable=f"{venv_path}/bin/python", - prefix="/Library/Frameworks/Python.framework/Versions/9.8", - exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8", + prefix=venv_path, + exec_prefix=venv_path, base_executable="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8", base_prefix="/Library/Frameworks/Python.framework/Versions/9.8", base_exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8", @@ -697,8 +697,8 @@ def test_venv_alt_framework_macos(self): ]) expected = dict( executable=f"{venv_path}/bin/python", - prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", - exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + prefix=venv_path, + exec_prefix=venv_path, base_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8", base_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", base_exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", @@ -734,8 +734,8 @@ def test_venv_macos(self): ]) expected = dict( executable="/framework/Python9.8/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/framework/Python9.8", + exec_prefix="/framework/Python9.8", base_executable="/usr/bin/python", base_prefix="/usr", base_exec_prefix="/usr", diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 9bbf8d0c6cf2da..a705dd0cd89e4d 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -110,6 +110,7 @@ def venv(self, **venv_create_args): **venv_create_args, ) + def test_get_path_names(self): self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS) @@ -591,71 +592,6 @@ def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') self.assertTrue(suffix.endswith('-darwin.so'), suffix) - @requires_subprocess() - def test_config_vars_depend_on_site_initialization(self): - script = textwrap.dedent(""" - import sysconfig - - config_vars = sysconfig.get_config_vars() - - import json - print(json.dumps(config_vars, indent=2)) - """) - - with self.venv() as venv: - site_config_vars = json.loads(venv.run('-c', script).stdout) - no_site_config_vars = json.loads(venv.run('-S', '-c', script).stdout) - - self.assertNotEqual(site_config_vars, no_site_config_vars) - # With the site initialization, the virtual environment should be enabled. - self.assertEqual(site_config_vars['base'], venv.prefix) - self.assertEqual(site_config_vars['platbase'], venv.prefix) - #self.assertEqual(site_config_vars['prefix'], venv.prefix) # # FIXME: prefix gets overwriten by _init_posix - # Without the site initialization, the virtual environment should be disabled. - self.assertEqual(no_site_config_vars['base'], site_config_vars['installed_base']) - self.assertEqual(no_site_config_vars['platbase'], site_config_vars['installed_platbase']) - - @requires_subprocess() - def test_config_vars_recalculation_after_site_initialization(self): - script = textwrap.dedent(""" - import sysconfig - - before = sysconfig.get_config_vars() - - import site - site.main() - - after = sysconfig.get_config_vars() - - import json - print(json.dumps({'before': before, 'after': after}, indent=2)) - """) - - with self.venv() as venv: - config_vars = json.loads(venv.run('-S', '-c', script).stdout) - - self.assertNotEqual(config_vars['before'], config_vars['after']) - self.assertEqual(config_vars['after']['base'], venv.prefix) - #self.assertEqual(config_vars['after']['prefix'], venv.prefix) # FIXME: prefix gets overwriten by _init_posix - #self.assertEqual(config_vars['after']['exec_prefix'], venv.prefix) # FIXME: exec_prefix gets overwriten by _init_posix - - @requires_subprocess() - def test_paths_depend_on_site_initialization(self): - script = textwrap.dedent(""" - import sysconfig - - paths = sysconfig.get_paths() - - import json - print(json.dumps(paths, indent=2)) - """) - - with self.venv() as venv: - site_paths = json.loads(venv.run('-c', script).stdout) - no_site_paths = json.loads(venv.run('-S', '-c', script).stdout) - - self.assertNotEqual(site_paths, no_site_paths) - @requires_subprocess() def test_makefile_overwrites_config_vars(self): script = textwrap.dedent(""" @@ -689,7 +625,6 @@ def test_makefile_overwrites_config_vars(self): self.assertNotEqual(data['prefix'], data['base_prefix']) self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) - class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), diff --git a/Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst b/Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst new file mode 100644 index 00000000000000..c875c7b547bba9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst @@ -0,0 +1,3 @@ +When running under a virtual environment with the :mod:`site` disabled (see +:option:`-S`), :data:`sys.prefix` and :data:`sys.base_prefix` will now point +to the virtual environment, instead of the base installation. diff --git a/Modules/getpath.py b/Modules/getpath.py index 1c1eb6cbf3ee22..7949fd813d0d07 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -640,11 +640,20 @@ def search_up(prefix, *landmarks, test=isfile): # For a venv, update the main prefix/exec_prefix but leave the base ones unchanged -# XXX: We currently do not update prefix here, but it happens in site.py -#if venv_prefix: -# base_prefix = prefix -# base_exec_prefix = exec_prefix -# prefix = exec_prefix = venv_prefix +if venv_prefix: + if not base_prefix: + base_prefix = prefix + if not base_exec_prefix: + base_exec_prefix = exec_prefix + prefix = exec_prefix = venv_prefix + + +# After calculating prefix and exec_prefix, use their values for base_prefix and +# base_exec_prefix if they haven't been set. +if not base_prefix: + base_prefix = prefix +if not base_exec_prefix: + base_exec_prefix = exec_prefix # ****************************************************************************** @@ -679,7 +688,7 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: POSIX uses the default prefix when in the build directory pythonpath.append(joinpath(PREFIX, ZIP_LANDMARK)) else: - pythonpath.append(joinpath(prefix, ZIP_LANDMARK)) + pythonpath.append(joinpath(base_prefix, ZIP_LANDMARK)) if os_name == 'nt' and use_environment and winreg: # QUIRK: Windows also lists paths in the registry. Paths are stored @@ -714,13 +723,13 @@ def search_up(prefix, *landmarks, test=isfile): # Then add any entries compiled into the PYTHONPATH macro. if PYTHONPATH: for p in PYTHONPATH.split(DELIM): - pythonpath.append(joinpath(prefix, p)) + pythonpath.append(joinpath(base_prefix, p)) # Then add stdlib_dir and platstdlib_dir - if not stdlib_dir and prefix: - stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) - if not platstdlib_dir and exec_prefix: - platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK) + if not stdlib_dir and base_prefix: + stdlib_dir = joinpath(base_prefix, STDLIB_SUBDIR) + if not platstdlib_dir and base_exec_prefix: + platstdlib_dir = joinpath(base_exec_prefix, PLATSTDLIB_LANDMARK) if os_name == 'nt': # QUIRK: Windows generates paths differently @@ -750,9 +759,13 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running # in build directory. This happens after pythonpath calculation. +# Virtual environments using the build directory Python still keep their prefix. if os_name != 'nt' and build_prefix: - prefix = config.get('prefix') or PREFIX - exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix + if not venv_prefix: + prefix = config.get('prefix') or PREFIX + exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix + base_prefix = config.get('base_prefix') or PREFIX + base_exec_prefix = config.get('base_exec_prefix') or EXEC_PREFIX or base_prefix # ****************************************************************************** @@ -788,8 +801,8 @@ def search_up(prefix, *landmarks, test=isfile): config['base_executable'] = base_executable config['prefix'] = prefix config['exec_prefix'] = exec_prefix -config['base_prefix'] = base_prefix or prefix -config['base_exec_prefix'] = base_exec_prefix or exec_prefix +config['base_prefix'] = base_prefix +config['base_exec_prefix'] = base_exec_prefix config['platlibdir'] = platlibdir # test_embed expects empty strings, not None From dcf629213bc046318c862ec0af5db3dfd1fc473a Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 26 Nov 2024 07:40:13 -0800 Subject: [PATCH 026/427] gh-119180: Add VALUE_WITH_FAKE_GLOBALS format to annotationlib (#124415) --- Doc/library/annotationlib.rst | 11 ++++++++++ Include/internal/pycore_object.h | 7 ++++++ Lib/annotationlib.py | 13 +++++++---- Lib/test/test_annotationlib.py | 36 ++++++++++++++++++++++--------- Lib/test/test_type_annotations.py | 4 ++-- Lib/typing.py | 15 ++++++++----- Objects/typevarobject.c | 4 ++-- Python/codegen.c | 9 +++++--- 8 files changed, 73 insertions(+), 26 deletions(-) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 37490456d13312..dcaff3d7fdbec5 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -144,6 +144,17 @@ Classes The exact values of these strings may change in future versions of Python. + .. attribute:: VALUE_WITH_FAKE_GLOBALS + :value: 4 + + Special value used to signal that an annotate function is being + evaluated in a special environment with fake globals. When passed this + value, annotate functions should either return the same value as for + the :attr:`Format.VALUE` format, or raise :exc:`NotImplementedError` + to signal that they do not support execution in this environment. + This format is only used internally and should not be passed to + the functions in this module. + .. versionadded:: 3.14 .. class:: ForwardRef diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 783d88cb51ffbd..34d835a7f84ee7 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -919,6 +919,13 @@ PyAPI_DATA(int) _Py_SwappedOp[]; extern void _Py_GetConstant_Init(void); +enum _PyAnnotateFormat { + _Py_ANNOTATE_FORMAT_VALUE = 1, + _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS = 2, + _Py_ANNOTATE_FORMAT_FORWARDREF = 3, + _Py_ANNOTATE_FORMAT_STRING = 4, +}; + #ifdef __cplusplus } #endif diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 732fbfa628cf5f..42f1f3877514d9 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -22,8 +22,9 @@ class Format(enum.IntEnum): VALUE = 1 - FORWARDREF = 2 - STRING = 3 + VALUE_WITH_FAKE_GLOBALS = 2 + FORWARDREF = 3 + STRING = 4 _Union = None @@ -513,6 +514,8 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): on the generated ForwardRef objects. """ + if format == Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") try: return annotate(format) except NotImplementedError: @@ -546,7 +549,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): argdefs=annotate.__defaults__, kwdefaults=annotate.__kwdefaults__, ) - annos = func(Format.VALUE) + annos = func(Format.VALUE_WITH_FAKE_GLOBALS) if _is_evaluate: return annos if isinstance(annos, str) else repr(annos) return { @@ -607,7 +610,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): argdefs=annotate.__defaults__, kwdefaults=annotate.__kwdefaults__, ) - result = func(Format.VALUE) + result = func(Format.VALUE_WITH_FAKE_GLOBALS) for obj in globals.stringifiers: obj.__class__ = ForwardRef obj.__stringifier_dict__ = None # not needed for ForwardRef @@ -726,6 +729,8 @@ def get_annotations( # But if we didn't get it, we use __annotations__ instead. ann = _get_dunder_annotations(obj) return annotations_to_string(ann) + case Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") case _: raise ValueError(f"Unsupported format {format!r}") diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 2ca7058c14398c..20f74b4ed0aadb 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -42,11 +42,14 @@ def test_enum(self): self.assertEqual(Format.VALUE.value, 1) self.assertEqual(Format.VALUE, 1) - self.assertEqual(Format.FORWARDREF.value, 2) - self.assertEqual(Format.FORWARDREF, 2) + self.assertEqual(Format.VALUE_WITH_FAKE_GLOBALS.value, 2) + self.assertEqual(Format.VALUE_WITH_FAKE_GLOBALS, 2) - self.assertEqual(Format.STRING.value, 3) - self.assertEqual(Format.STRING, 3) + self.assertEqual(Format.FORWARDREF.value, 3) + self.assertEqual(Format.FORWARDREF, 3) + + self.assertEqual(Format.STRING.value, 4) + self.assertEqual(Format.STRING, 4) class TestForwardRefFormat(unittest.TestCase): @@ -459,19 +462,28 @@ def f2(a: undefined): annotationlib.get_annotations(f2, format=Format.FORWARDREF), {"a": fwd}, ) - self.assertEqual(annotationlib.get_annotations(f2, format=2), {"a": fwd}) + self.assertEqual(annotationlib.get_annotations(f2, format=3), {"a": fwd}) self.assertEqual( annotationlib.get_annotations(f1, format=Format.STRING), {"a": "int"}, ) - self.assertEqual(annotationlib.get_annotations(f1, format=3), {"a": "int"}) + self.assertEqual(annotationlib.get_annotations(f1, format=4), {"a": "int"}) with self.assertRaises(ValueError): - annotationlib.get_annotations(f1, format=0) + annotationlib.get_annotations(f1, format=42) - with self.assertRaises(ValueError): - annotationlib.get_annotations(f1, format=4) + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + annotationlib.get_annotations(f1, format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + annotationlib.get_annotations(f1, format=2) def test_custom_object_with_annotations(self): class C: @@ -505,6 +517,8 @@ def foo(a: int, b: str): foo.__annotations__ = {"a": "foo", "b": "str"} for format in Format: + if format is Format.VALUE_WITH_FAKE_GLOBALS: + continue with self.subTest(format=format): self.assertEqual( annotationlib.get_annotations(foo, format=format), @@ -802,6 +816,8 @@ def __annotations__(self): wa = WeirdAnnotations() for format in Format: + if format is Format.VALUE_WITH_FAKE_GLOBALS: + continue with ( self.subTest(format=format), self.assertRaisesRegex( @@ -990,7 +1006,7 @@ def test_pep_695_generics_with_future_annotations_nested_in_function(self): class TestCallEvaluateFunction(unittest.TestCase): def test_evaluation(self): def evaluate(format, exc=NotImplementedError): - if format != 1: + if format > 2: raise exc return undefined diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 257b7fa95dcb76..7d88f4cdfa3141 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -316,7 +316,7 @@ def test_module(self): ns = run_code("x: undefined = 1") anno = ns["__annotate__"] with self.assertRaises(NotImplementedError): - anno(2) + anno(3) with self.assertRaises(NameError): anno(1) @@ -376,7 +376,7 @@ class X: annotate(annotationlib.Format.FORWARDREF) with self.assertRaises(NotImplementedError): annotate(annotationlib.Format.STRING) - with self.assertRaises(NotImplementedError): + with self.assertRaises(TypeError): annotate(None) self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) diff --git a/Lib/typing.py b/Lib/typing.py index 938e52922aee03..5f3aacd877221c 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2936,10 +2936,13 @@ def _make_eager_annotate(types): checked_types = {key: _type_check(val, f"field {key} annotation must be a type") for key, val in types.items()} def annotate(format): - if format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): - return checked_types - else: - return annotationlib.annotations_to_string(types) + match format: + case annotationlib.Format.VALUE | annotationlib.Format.FORWARDREF: + return checked_types + case annotationlib.Format.STRING: + return annotationlib.annotations_to_string(types) + case _: + raise NotImplementedError(format) return annotate @@ -3229,8 +3232,10 @@ def __annotate__(format): } elif format == annotationlib.Format.STRING: own = annotationlib.annotations_to_string(own_annotations) - else: + elif format in (annotationlib.Format.FORWARDREF, annotationlib.Format.VALUE): own = own_checked_annotations + else: + raise NotImplementedError(format) annos.update(own) return annos diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index bacb858978c5d7..4ed40aa71a595e 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -1,6 +1,6 @@ // TypeVar, TypeVarTuple, and ParamSpec #include "Python.h" -#include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK +#include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK, PyAnnotateFormat #include "pycore_typevarobject.h" #include "pycore_unionobject.h" // _Py_union_type_or @@ -168,7 +168,7 @@ constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } PyObject *value = ((constevaluatorobject *)self)->value; - if (format == 3) { // STRING + if (format == _Py_ANNOTATE_FORMAT_STRING) { PyUnicodeWriter *writer = PyUnicodeWriter_Create(5); // cannot be <5 if (writer == NULL) { return NULL; diff --git a/Python/codegen.c b/Python/codegen.c index dbf36cdc0b7908..a5e550cf8c947e 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -24,6 +24,7 @@ #include "pycore_instruction_sequence.h" // _PyInstructionSequence_NewLabel() #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_object.h" // _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS #include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_symtable.h" // PySTEntryObject @@ -672,14 +673,16 @@ codegen_setup_annotations_scope(compiler *c, location loc, codegen_enter_scope(c, name, COMPILE_SCOPE_ANNOTATIONS, key, loc.lineno, NULL, &umd)); + // if .format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + PyObject *value_with_fake_globals = PyLong_FromLong(_Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS); assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); - // if .format != 1: raise NotImplementedError _Py_DECLARE_STR(format, ".format"); ADDOP_I(c, loc, LOAD_FAST, 0); - ADDOP_LOAD_CONST(c, loc, _PyLong_GetOne()); - ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]); + ADDOP_LOAD_CONST(c, loc, value_with_fake_globals); + ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]); NEW_JUMP_TARGET_LABEL(c, body); ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); + ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); ADDOP_I(c, loc, RAISE_VARARGS, 1); USE_LABEL(c, body); From 987311d42e3ec838de8ff27f9f0575aa791a6bde Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 26 Nov 2024 18:57:39 +0300 Subject: [PATCH 027/427] gh-69639: Add mixed-mode rules for complex arithmetic (C-like) (GH-124829) "Generally, mixed-mode arithmetic combining real and complex variables should be performed directly, not by first coercing the real to complex, lest the sign of zero be rendered uninformative; the same goes for combinations of pure imaginary quantities with complex variables." (c) Kahan, W: Branch cuts for complex elementary functions. This patch implements mixed-mode arithmetic rules, combining real and complex variables as specified by C standards since C99 (in particular, there is no special version for the true division with real lhs operand). Most C compilers implementing C99+ Annex G have only these special rules (without support for imaginary type, which is going to be deprecated in C2y). --- Doc/c-api/complex.rst | 54 ++++ Doc/library/cmath.rst | 12 +- Doc/library/stdtypes.rst | 16 +- Doc/reference/expressions.rst | 25 +- Doc/whatsnew/3.14.rst | 4 + Include/cpython/complexobject.h | 6 + Include/internal/pycore_floatobject.h | 2 + Lib/test/test_builtin.py | 14 +- Lib/test/test_capi/test_complex.py | 45 +++- Lib/test/test_complex.py | 57 +++++ ...4-08-03-14-02-27.gh-issue-69639.mW3iKq.rst | 2 + Modules/_testcapi/complex.c | 50 +++- Objects/complexobject.c | 236 +++++++++++++----- Objects/floatobject.c | 12 +- Python/bltinmodule.c | 2 - 15 files changed, 444 insertions(+), 93 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-08-03-14-02-27.gh-issue-69639.mW3iKq.rst diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 16bd79475dc1e6..d1f5d8eda676ef 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -44,12 +44,36 @@ pointers. This is consistent throughout the API. representation. +.. c:function:: Py_complex _Py_cr_sum(Py_complex left, double right) + + Return the sum of a complex number and a real number, using the C :c:type:`Py_complex` + representation. + + .. versionadded:: 3.14 + + .. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) Return the difference between two complex numbers, using the C :c:type:`Py_complex` representation. +.. c:function:: Py_complex _Py_cr_diff(Py_complex left, double right) + + Return the difference between a complex number and a real number, using the C + :c:type:`Py_complex` representation. + + .. versionadded:: 3.14 + + +.. c:function:: Py_complex _Py_rc_diff(double left, Py_complex right) + + Return the difference between a real number and a complex number, using the C + :c:type:`Py_complex` representation. + + .. versionadded:: 3.14 + + .. c:function:: Py_complex _Py_c_neg(Py_complex num) Return the negation of the complex number *num*, using the C @@ -62,6 +86,14 @@ pointers. This is consistent throughout the API. representation. +.. c:function:: Py_complex _Py_cr_prod(Py_complex left, double right) + + Return the product of a complex number and a real number, using the C + :c:type:`Py_complex` representation. + + .. versionadded:: 3.14 + + .. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) Return the quotient of two complex numbers, using the C :c:type:`Py_complex` @@ -71,6 +103,28 @@ pointers. This is consistent throughout the API. :c:data:`errno` to :c:macro:`!EDOM`. +.. c:function:: Py_complex _Py_cr_quot(Py_complex dividend, double divisor) + + Return the quotient of a complex number and a real number, using the C + :c:type:`Py_complex` representation. + + If *divisor* is zero, this method returns zero and sets + :c:data:`errno` to :c:macro:`!EDOM`. + + .. versionadded:: 3.14 + + +.. c:function:: Py_complex _Py_rc_quot(double dividend, Py_complex divisor) + + Return the quotient of a real number and a complex number, using the C + :c:type:`Py_complex` representation. + + If *divisor* is zero, this method returns zero and sets + :c:data:`errno` to :c:macro:`!EDOM`. + + .. versionadded:: 3.14 + + .. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index f122e3644ece56..e7c027dd4d0c22 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -24,17 +24,17 @@ the function is then applied to the result of the conversion. imaginary axis we look at the sign of the real part. For example, the :func:`cmath.sqrt` function has a branch cut along the - negative real axis. An argument of ``complex(-2.0, -0.0)`` is treated as + negative real axis. An argument of ``-2-0j`` is treated as though it lies *below* the branch cut, and so gives a result on the negative imaginary axis:: - >>> cmath.sqrt(complex(-2.0, -0.0)) + >>> cmath.sqrt(-2-0j) -1.4142135623730951j - But an argument of ``complex(-2.0, 0.0)`` is treated as though it lies above + But an argument of ``-2+0j`` is treated as though it lies above the branch cut:: - >>> cmath.sqrt(complex(-2.0, 0.0)) + >>> cmath.sqrt(-2+0j) 1.4142135623730951j @@ -63,9 +63,9 @@ rectangular coordinates to polar coordinates and back. along the negative real axis. The sign of the result is the same as the sign of ``x.imag``, even when ``x.imag`` is zero:: - >>> phase(complex(-1.0, 0.0)) + >>> phase(-1+0j) 3.141592653589793 - >>> phase(complex(-1.0, -0.0)) + >>> phase(-1-0j) -3.141592653589793 diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 2347437d7273d9..4f4fc9fba63120 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -243,6 +243,9 @@ numeric literal yields an imaginary number (a complex number with a zero real part) which you can add to an integer or float to get a complex number with real and imaginary parts. +The constructors :func:`int`, :func:`float`, and +:func:`complex` can be used to produce numbers of a specific type. + .. index:: single: arithmetic pair: built-in function; int @@ -262,12 +265,15 @@ and imaginary parts. Python fully supports mixed arithmetic: when a binary arithmetic operator has operands of different numeric types, the operand with the "narrower" type is -widened to that of the other, where integer is narrower than floating point, -which is narrower than complex. A comparison between numbers of different types -behaves as though the exact values of those numbers were being compared. [2]_ +widened to that of the other, where integer is narrower than floating point. +Arithmetic with complex and real operands is defined by the usual mathematical +formula, for example:: -The constructors :func:`int`, :func:`float`, and -:func:`complex` can be used to produce numbers of a specific type. + x + complex(u, v) = complex(x + u, v) + x * complex(u, v) = complex(x * u, x * v) + +A comparison between numbers of different types behaves as though the exact +values of those numbers were being compared. [2]_ All numeric types (except complex) support the following operations (for priorities of the operations, see :ref:`operator-summary`): diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 3eaceae41f7eaf..7c95b207b1aed2 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -28,13 +28,12 @@ Arithmetic conversions .. index:: pair: arithmetic; conversion When a description of an arithmetic operator below uses the phrase "the numeric -arguments are converted to a common type", this means that the operator +arguments are converted to a common real type", this means that the operator implementation for built-in types works as follows: -* If either argument is a complex number, the other is converted to complex; +* If both arguments are complex numbers, no conversion is performed; -* otherwise, if either argument is a floating-point number, the other is - converted to floating point; +* if either argument is a complex or a floating-point number, the other is converted to a floating-point number; * otherwise, both must be integers and no conversion is necessary. @@ -1323,12 +1322,16 @@ operators and one for additive operators: The ``*`` (multiplication) operator yields the product of its arguments. The arguments must either both be numbers, or one argument must be an integer and the other must be a sequence. In the former case, the numbers are converted to a -common type and then multiplied together. In the latter case, sequence +common real type and then multiplied together. In the latter case, sequence repetition is performed; a negative repetition factor yields an empty sequence. This operation can be customized using the special :meth:`~object.__mul__` and :meth:`~object.__rmul__` methods. +.. versionchanged:: 3.14 + If only one operand is a complex number, the other operand is converted + to a floating-point number. + .. index:: single: matrix multiplication pair: operator; @ (at) @@ -1396,23 +1399,31 @@ floating-point number using the :func:`abs` function if appropriate. The ``+`` (addition) operator yields the sum of its arguments. The arguments must either both be numbers or both be sequences of the same type. In the -former case, the numbers are converted to a common type and then added together. +former case, the numbers are converted to a common real type and then added together. In the latter case, the sequences are concatenated. This operation can be customized using the special :meth:`~object.__add__` and :meth:`~object.__radd__` methods. +.. versionchanged:: 3.14 + If only one operand is a complex number, the other operand is converted + to a floating-point number. + .. index:: single: subtraction single: operator; - (minus) single: - (minus); binary operator The ``-`` (subtraction) operator yields the difference of its arguments. The -numeric arguments are first converted to a common type. +numeric arguments are first converted to a common real type. This operation can be customized using the special :meth:`~object.__sub__` and :meth:`~object.__rsub__` methods. +.. versionchanged:: 3.14 + If only one operand is a complex number, the other operand is converted + to a floating-point number. + .. _shifting: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 0e4b9eb0cf0b9c..869a47c1261293 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -230,6 +230,10 @@ Other language changes They raise an error if the argument is a string. (Contributed by Serhiy Storchaka in :gh:`84978`.) +* Implement mixed-mode arithmetic rules combining real and complex numbers as + specified by C standards since C99. + (Contributed by Sergey B Kirpichev in :gh:`69639`.) + * All Windows code pages are now supported as "cpXXX" codecs on Windows. (Contributed by Serhiy Storchaka in :gh:`123803`.) diff --git a/Include/cpython/complexobject.h b/Include/cpython/complexobject.h index fbdc6a91fe895c..28576afad0b6b5 100644 --- a/Include/cpython/complexobject.h +++ b/Include/cpython/complexobject.h @@ -9,10 +9,16 @@ typedef struct { // Operations on complex numbers. PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_cr_sum(Py_complex, double); PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_cr_diff(Py_complex, double); +PyAPI_FUNC(Py_complex) _Py_rc_diff(double, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_cr_prod(Py_complex, double); PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_cr_quot(Py_complex, double); +PyAPI_FUNC(Py_complex) _Py_rc_quot(double, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); PyAPI_FUNC(double) _Py_c_abs(Py_complex); diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index be1c6cc97720d2..f44b081b06cea5 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -54,6 +54,8 @@ extern PyObject* _Py_string_to_number_with_underscores( extern double _Py_parse_inf_or_nan(const char *p, char **endptr); +extern int _Py_convert_int_to_double(PyObject **v, double *dbl); + #ifdef __cplusplus } diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index f8e6f05cd607c8..e51711d9b4f1a4 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -35,6 +35,7 @@ from test.support.import_helper import import_module from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok +from test.support.testcase import ComplexesAreIdenticalMixin from test.support.warnings_helper import check_warnings from test.support import requires_IEEE_754 from unittest.mock import MagicMock, patch @@ -151,7 +152,7 @@ def map_char(arg): def pack(*args): return args -class BuiltinTest(unittest.TestCase): +class BuiltinTest(ComplexesAreIdenticalMixin, unittest.TestCase): # Helper to check picklability def check_iter_pickle(self, it, seq, proto): itorg = it @@ -1902,6 +1903,17 @@ def __getitem__(self, index): self.assertEqual(sum(xs), complex(sum(z.real for z in xs), sum(z.imag for z in xs))) + # test that sum() of complex and real numbers doesn't + # smash sign of imaginary 0 + self.assertComplexesAreIdentical(sum([complex(1, -0.0), 1]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([1, complex(1, -0.0)]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([complex(1, -0.0), 1.0]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([1.0, complex(1, -0.0)]), + complex(2, -0.0)) + @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "sum accuracy not guaranteed on machines with double rounding") diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py index 368edfbf2ce97e..97e0eb3f043080 100644 --- a/Lib/test/test_capi/test_complex.py +++ b/Lib/test/test_capi/test_complex.py @@ -7,6 +7,7 @@ FloatSubclass, Float, BadFloat, BadFloat2, ComplexSubclass) from test.support import import_helper +from test.support.testcase import ComplexesAreIdenticalMixin _testcapi = import_helper.import_module('_testcapi') @@ -23,7 +24,7 @@ def __complex__(self): raise RuntimeError -class CAPIComplexTest(unittest.TestCase): +class CAPIComplexTest(ComplexesAreIdenticalMixin, unittest.TestCase): def test_check(self): # Test PyComplex_Check() check = _testlimitedcapi.complex_check @@ -171,12 +172,33 @@ def test_py_c_sum(self): self.assertEqual(_py_c_sum(1, 1j), (1+1j, 0)) + def test_py_cr_sum(self): + # Test _Py_cr_sum() + _py_cr_sum = _testcapi._py_cr_sum + + self.assertComplexesAreIdentical(_py_cr_sum(-0j, -0.0)[0], + complex(-0.0, -0.0)) + def test_py_c_diff(self): # Test _Py_c_diff() _py_c_diff = _testcapi._py_c_diff self.assertEqual(_py_c_diff(1, 1j), (1-1j, 0)) + def test_py_cr_diff(self): + # Test _Py_cr_diff() + _py_cr_diff = _testcapi._py_cr_diff + + self.assertComplexesAreIdentical(_py_cr_diff(-0j, 0.0)[0], + complex(-0.0, -0.0)) + + def test_py_rc_diff(self): + # Test _Py_rc_diff() + _py_rc_diff = _testcapi._py_rc_diff + + self.assertComplexesAreIdentical(_py_rc_diff(-0.0, 0j)[0], + complex(-0.0, -0.0)) + def test_py_c_neg(self): # Test _Py_c_neg() _py_c_neg = _testcapi._py_c_neg @@ -189,6 +211,13 @@ def test_py_c_prod(self): self.assertEqual(_py_c_prod(2, 1j), (2j, 0)) + def test_py_cr_prod(self): + # Test _Py_cr_prod() + _py_cr_prod = _testcapi._py_cr_prod + + self.assertComplexesAreIdentical(_py_cr_prod(complex('inf+1j'), INF)[0], + complex('inf+infj')) + def test_py_c_quot(self): # Test _Py_c_quot() _py_c_quot = _testcapi._py_c_quot @@ -211,6 +240,20 @@ def test_py_c_quot(self): self.assertEqual(_py_c_quot(1, 0j)[1], errno.EDOM) + def test_py_cr_quot(self): + # Test _Py_cr_quot() + _py_cr_quot = _testcapi._py_cr_quot + + self.assertComplexesAreIdentical(_py_cr_quot(complex('inf+1j'), 2**1000)[0], + INF + 2**-1000*1j) + + def test_py_rc_quot(self): + # Test _Py_rc_quot() + _py_rc_quot = _testcapi._py_rc_quot + + self.assertComplexesAreIdentical(_py_rc_quot(1.0, complex('nan-infj'))[0], + 0j) + def test_py_c_pow(self): # Test _Py_c_pow() _py_c_pow = _testcapi._py_c_pow diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index c51327c7f33a0a..179556f57e884f 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -126,6 +126,12 @@ def test_truediv(self): z = complex(0, 0) / complex(denom_real, denom_imag) self.assertTrue(isnan(z.real)) self.assertTrue(isnan(z.imag)) + z = float(0) / complex(denom_real, denom_imag) + self.assertTrue(isnan(z.real)) + self.assertTrue(isnan(z.imag)) + + self.assertComplexesAreIdentical(complex(INF, NAN) / 2, + complex(INF, NAN)) self.assertComplexesAreIdentical(complex(INF, 1)/(0.0+1j), complex(NAN, -INF)) @@ -154,6 +160,39 @@ def test_truediv(self): self.assertComplexesAreIdentical(complex(INF, 1)/complex(1, INF), complex(NAN, NAN)) + # mixed types + self.assertEqual((1+1j)/float(2), 0.5+0.5j) + self.assertEqual(float(1)/(1+2j), 0.2-0.4j) + self.assertEqual(float(1)/(-1+2j), -0.2-0.4j) + self.assertEqual(float(1)/(1-2j), 0.2+0.4j) + self.assertEqual(float(1)/(2+1j), 0.4-0.2j) + self.assertEqual(float(1)/(-2+1j), -0.4-0.2j) + self.assertEqual(float(1)/(2-1j), 0.4+0.2j) + + self.assertComplexesAreIdentical(INF/(1+0j), + complex(INF, NAN)) + self.assertComplexesAreIdentical(INF/(0.0+1j), + complex(NAN, -INF)) + self.assertComplexesAreIdentical(INF/complex(2**1000, 2**-1000), + complex(INF, NAN)) + self.assertComplexesAreIdentical(INF/complex(NAN, NAN), + complex(NAN, NAN)) + + self.assertComplexesAreIdentical(float(1)/complex(INF, INF), (0.0-0j)) + self.assertComplexesAreIdentical(float(1)/complex(INF, -INF), (0.0+0j)) + self.assertComplexesAreIdentical(float(1)/complex(-INF, INF), + complex(-0.0, -0.0)) + self.assertComplexesAreIdentical(float(1)/complex(-INF, -INF), + complex(-0.0, 0)) + self.assertComplexesAreIdentical(float(1)/complex(INF, NAN), + complex(0.0, -0.0)) + self.assertComplexesAreIdentical(float(1)/complex(-INF, NAN), + complex(-0.0, -0.0)) + self.assertComplexesAreIdentical(float(1)/complex(NAN, INF), + complex(0.0, -0.0)) + self.assertComplexesAreIdentical(float(INF)/complex(NAN, INF), + complex(NAN, NAN)) + def test_truediv_zero_division(self): for a, b in ZERO_DIVISION: with self.assertRaises(ZeroDivisionError): @@ -224,6 +263,10 @@ def check(n, deltas, is_equal, imag = 0.0): def test_add(self): self.assertEqual(1j + int(+1), complex(+1, 1)) self.assertEqual(1j + int(-1), complex(-1, 1)) + self.assertComplexesAreIdentical(complex(-0.0, -0.0) + (-0.0), + complex(-0.0, -0.0)) + self.assertComplexesAreIdentical((-0.0) + complex(-0.0, -0.0), + complex(-0.0, -0.0)) self.assertRaises(OverflowError, operator.add, 1j, 10**1000) self.assertRaises(TypeError, operator.add, 1j, None) self.assertRaises(TypeError, operator.add, None, 1j) @@ -231,6 +274,14 @@ def test_add(self): def test_sub(self): self.assertEqual(1j - int(+1), complex(-1, 1)) self.assertEqual(1j - int(-1), complex(1, 1)) + self.assertComplexesAreIdentical(complex(-0.0, -0.0) - 0.0, + complex(-0.0, -0.0)) + self.assertComplexesAreIdentical(-0.0 - complex(0.0, 0.0), + complex(-0.0, -0.0)) + self.assertComplexesAreIdentical(complex(1, 2) - complex(2, 1), + complex(-1, 1)) + self.assertComplexesAreIdentical(complex(2, 1) - complex(1, 2), + complex(1, -1)) self.assertRaises(OverflowError, operator.sub, 1j, 10**1000) self.assertRaises(TypeError, operator.sub, 1j, None) self.assertRaises(TypeError, operator.sub, None, 1j) @@ -238,6 +289,12 @@ def test_sub(self): def test_mul(self): self.assertEqual(1j * int(20), complex(0, 20)) self.assertEqual(1j * int(-1), complex(0, -1)) + for c, r in [(2, complex(INF, 2)), (INF, complex(INF, INF)), + (0, complex(NAN, 0)), (-0.0, complex(NAN, -0.0)), + (NAN, complex(NAN, NAN))]: + with self.subTest(c=c, r=r): + self.assertComplexesAreIdentical(complex(INF, 1) * c, r) + self.assertComplexesAreIdentical(c * complex(INF, 1), r) self.assertRaises(OverflowError, operator.mul, 1j, 10**1000) self.assertRaises(TypeError, operator.mul, 1j, None) self.assertRaises(TypeError, operator.mul, None, 1j) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-03-14-02-27.gh-issue-69639.mW3iKq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-03-14-02-27.gh-issue-69639.mW3iKq.rst new file mode 100644 index 00000000000000..72596b0302aa45 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-03-14-02-27.gh-issue-69639.mW3iKq.rst @@ -0,0 +1,2 @@ +Implement mixed-mode arithmetic rules combining real and complex numbers +as specified by C standards since C99. Patch by Sergey B Kirpichev. diff --git a/Modules/_testcapi/complex.c b/Modules/_testcapi/complex.c index eceb1310bfe874..b726cd3236f179 100644 --- a/Modules/_testcapi/complex.c +++ b/Modules/_testcapi/complex.c @@ -46,21 +46,59 @@ _py_c_neg(PyObject *Py_UNUSED(module), PyObject *num) static PyObject * \ _py_c_##suffix(PyObject *Py_UNUSED(module), PyObject *args) \ { \ - Py_complex num, exp, res; \ + Py_complex a, b, res; \ \ - if (!PyArg_ParseTuple(args, "DD", &num, &exp)) { \ + if (!PyArg_ParseTuple(args, "DD", &a, &b)) { \ return NULL; \ } \ \ errno = 0; \ - res = _Py_c_##suffix(num, exp); \ + res = _Py_c_##suffix(a, b); \ + return Py_BuildValue("Di", &res, errno); \ + }; + +#define _PY_CR_FUNC2(suffix) \ + static PyObject * \ + _py_cr_##suffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + Py_complex a, res; \ + double b; \ + \ + if (!PyArg_ParseTuple(args, "Dd", &a, &b)) { \ + return NULL; \ + } \ + \ + errno = 0; \ + res = _Py_cr_##suffix(a, b); \ + return Py_BuildValue("Di", &res, errno); \ + }; + +#define _PY_RC_FUNC2(suffix) \ + static PyObject * \ + _py_rc_##suffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + Py_complex b, res; \ + double a; \ + \ + if (!PyArg_ParseTuple(args, "dD", &a, &b)) { \ + return NULL; \ + } \ + \ + errno = 0; \ + res = _Py_rc_##suffix(a, b); \ return Py_BuildValue("Di", &res, errno); \ }; _PY_C_FUNC2(sum) +_PY_CR_FUNC2(sum) _PY_C_FUNC2(diff) +_PY_CR_FUNC2(diff) +_PY_RC_FUNC2(diff) _PY_C_FUNC2(prod) +_PY_CR_FUNC2(prod) _PY_C_FUNC2(quot) +_PY_CR_FUNC2(quot) +_PY_RC_FUNC2(quot) _PY_C_FUNC2(pow) static PyObject* @@ -86,10 +124,16 @@ static PyMethodDef test_methods[] = { {"complex_fromccomplex", complex_fromccomplex, METH_O}, {"complex_asccomplex", complex_asccomplex, METH_O}, {"_py_c_sum", _py_c_sum, METH_VARARGS}, + {"_py_cr_sum", _py_cr_sum, METH_VARARGS}, {"_py_c_diff", _py_c_diff, METH_VARARGS}, + {"_py_cr_diff", _py_cr_diff, METH_VARARGS}, + {"_py_rc_diff", _py_rc_diff, METH_VARARGS}, {"_py_c_neg", _py_c_neg, METH_O}, {"_py_c_prod", _py_c_prod, METH_VARARGS}, + {"_py_cr_prod", _py_cr_prod, METH_VARARGS}, {"_py_c_quot", _py_c_quot, METH_VARARGS}, + {"_py_cr_quot", _py_cr_quot, METH_VARARGS}, + {"_py_rc_quot", _py_rc_quot, METH_VARARGS}, {"_py_c_pow", _py_c_pow, METH_VARARGS}, {"_py_c_abs", _py_c_abs, METH_O}, {NULL}, diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 9faa57519a424b..8fbca3cb02d80a 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -8,6 +8,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_complexobject.h" // _PyComplex_FormatAdvancedWriter() +#include "pycore_floatobject.h" // _Py_convert_int_to_double() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_object.h" // _PyObject_Init() #include "pycore_pymath.h" // _Py_ADJUST_ERANGE2() @@ -34,6 +35,20 @@ _Py_c_sum(Py_complex a, Py_complex b) return r; } +Py_complex +_Py_cr_sum(Py_complex a, double b) +{ + Py_complex r = a; + r.real += b; + return r; +} + +static inline Py_complex +_Py_rc_sum(double a, Py_complex b) +{ + return _Py_cr_sum(b, a); +} + Py_complex _Py_c_diff(Py_complex a, Py_complex b) { @@ -43,6 +58,23 @@ _Py_c_diff(Py_complex a, Py_complex b) return r; } +Py_complex +_Py_cr_diff(Py_complex a, double b) +{ + Py_complex r = a; + r.real -= b; + return r; +} + +Py_complex +_Py_rc_diff(double a, Py_complex b) +{ + Py_complex r; + r.real = a - b.real; + r.imag = -b.imag; + return r; +} + Py_complex _Py_c_neg(Py_complex a) { @@ -61,6 +93,21 @@ _Py_c_prod(Py_complex a, Py_complex b) return r; } +Py_complex +_Py_cr_prod(Py_complex a, double b) +{ + Py_complex r = a; + r.real *= b; + r.imag *= b; + return r; +} + +static inline Py_complex +_Py_rc_prod(double a, Py_complex b) +{ + return _Py_cr_prod(b, a); +} + /* Avoid bad optimization on Windows ARM64 until the compiler is fixed */ #ifdef _M_ARM64 #pragma optimize("", off) @@ -143,6 +190,64 @@ _Py_c_quot(Py_complex a, Py_complex b) return r; } + +Py_complex +_Py_cr_quot(Py_complex a, double b) +{ + Py_complex r = a; + if (b) { + r.real /= b; + r.imag /= b; + } + else { + errno = EDOM; + r.real = r.imag = 0.0; + } + return r; +} + +/* an equivalent of _Py_c_quot() function, when 1st argument is real */ +Py_complex +_Py_rc_quot(double a, Py_complex b) +{ + Py_complex r; + const double abs_breal = b.real < 0 ? -b.real : b.real; + const double abs_bimag = b.imag < 0 ? -b.imag : b.imag; + + if (abs_breal >= abs_bimag) { + if (abs_breal == 0.0) { + errno = EDOM; + r.real = r.imag = 0.0; + } + else { + const double ratio = b.imag / b.real; + const double denom = b.real + b.imag * ratio; + r.real = a / denom; + r.imag = (-a * ratio) / denom; + } + } + else if (abs_bimag >= abs_breal) { + const double ratio = b.real / b.imag; + const double denom = b.real * ratio + b.imag; + assert(b.imag != 0.0); + r.real = (a * ratio) / denom; + r.imag = (-a) / denom; + } + else { + r.real = r.imag = Py_NAN; + } + + if (isnan(r.real) && isnan(r.imag) && isfinite(a) + && (isinf(abs_breal) || isinf(abs_bimag))) + { + const double x = copysign(isinf(b.real) ? 1.0 : 0.0, b.real); + const double y = copysign(isinf(b.imag) ? 1.0 : 0.0, b.imag); + r.real = 0.0 * (a*x); + r.imag = 0.0 * (-a*y); + } + + return r; +} #ifdef _M_ARM64 #pragma optimize("", on) #endif @@ -474,83 +579,90 @@ complex_hash(PyComplexObject *v) } /* This macro may return! */ -#define TO_COMPLEX(obj, c) \ - if (PyComplex_Check(obj)) \ - c = ((PyComplexObject *)(obj))->cval; \ - else if (to_complex(&(obj), &(c)) < 0) \ +#define TO_COMPLEX(obj, c) \ + if (PyComplex_Check(obj)) \ + c = ((PyComplexObject *)(obj))->cval; \ + else if (real_to_complex(&(obj), &(c)) < 0) \ return (obj) static int -to_complex(PyObject **pobj, Py_complex *pc) +real_to_double(PyObject **pobj, double *dbl) { PyObject *obj = *pobj; - pc->real = pc->imag = 0.0; - if (PyLong_Check(obj)) { - pc->real = PyLong_AsDouble(obj); - if (pc->real == -1.0 && PyErr_Occurred()) { - *pobj = NULL; - return -1; - } - return 0; - } if (PyFloat_Check(obj)) { - pc->real = PyFloat_AsDouble(obj); - return 0; + *dbl = PyFloat_AS_DOUBLE(obj); + } + else if (_Py_convert_int_to_double(pobj, dbl) < 0) { + return -1; } - *pobj = Py_NewRef(Py_NotImplemented); - return -1; + return 0; } - -static PyObject * -complex_add(PyObject *v, PyObject *w) +static int +real_to_complex(PyObject **pobj, Py_complex *pc) { - Py_complex result; - Py_complex a, b; - TO_COMPLEX(v, a); - TO_COMPLEX(w, b); - result = _Py_c_sum(a, b); - return PyComplex_FromCComplex(result); + pc->imag = 0.0; + return real_to_double(pobj, &(pc->real)); } -static PyObject * -complex_sub(PyObject *v, PyObject *w) -{ - Py_complex result; - Py_complex a, b; - TO_COMPLEX(v, a); - TO_COMPLEX(w, b); - result = _Py_c_diff(a, b); - return PyComplex_FromCComplex(result); -} +/* Complex arithmetic rules implement special mixed-mode case where combining + a pure-real (float or int) value and a complex value is performed directly + without first coercing the real value to a complex value. -static PyObject * -complex_mul(PyObject *v, PyObject *w) -{ - Py_complex result; - Py_complex a, b; - TO_COMPLEX(v, a); - TO_COMPLEX(w, b); - result = _Py_c_prod(a, b); - return PyComplex_FromCComplex(result); -} + Let us consider the addition as an example, assuming that ints are implicitly + converted to floats. We have the following rules (up to variants with changed + order of operands): -static PyObject * -complex_div(PyObject *v, PyObject *w) -{ - Py_complex quot; - Py_complex a, b; - TO_COMPLEX(v, a); - TO_COMPLEX(w, b); - errno = 0; - quot = _Py_c_quot(a, b); - if (errno == EDOM) { - PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); - return NULL; - } - return PyComplex_FromCComplex(quot); -} + complex(a, b) + complex(c, d) = complex(a + c, b + d) + float(a) + complex(b, c) = complex(a + b, c) + + Similar rules are implemented for subtraction, multiplication and division. + See C11's Annex G, sections G.5.1 and G.5.2. + */ + +#define COMPLEX_BINOP(NAME, FUNC) \ + static PyObject * \ + complex_##NAME(PyObject *v, PyObject *w) \ + { \ + Py_complex a; \ + errno = 0; \ + if (PyComplex_Check(w)) { \ + Py_complex b = ((PyComplexObject *)w)->cval; \ + if (PyComplex_Check(v)) { \ + a = ((PyComplexObject *)v)->cval; \ + a = _Py_c_##FUNC(a, b); \ + } \ + else if (real_to_double(&v, &a.real) < 0) { \ + return v; \ + } \ + else { \ + a = _Py_rc_##FUNC(a.real, b); \ + } \ + } \ + else if (!PyComplex_Check(v)) { \ + Py_RETURN_NOTIMPLEMENTED; \ + } \ + else { \ + a = ((PyComplexObject *)v)->cval; \ + double b; \ + if (real_to_double(&w, &b) < 0) { \ + return w; \ + } \ + a = _Py_cr_##FUNC(a, b); \ + } \ + if (errno == EDOM) { \ + PyErr_SetString(PyExc_ZeroDivisionError, \ + "division by zero"); \ + return NULL; \ + } \ + return PyComplex_FromCComplex(a); \ + } + +COMPLEX_BINOP(add, sum) +COMPLEX_BINOP(mul, prod) +COMPLEX_BINOP(sub, diff) +COMPLEX_BINOP(div, quot) static PyObject * complex_pow(PyObject *v, PyObject *w, PyObject *z) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index f00b6a6b4b2bdc..bcc77287454768 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -341,16 +341,16 @@ PyFloat_AsDouble(PyObject *op) obj is not of float or int type, Py_NotImplemented is incref'ed, stored in obj, and returned from the function invoking this macro. */ -#define CONVERT_TO_DOUBLE(obj, dbl) \ - if (PyFloat_Check(obj)) \ - dbl = PyFloat_AS_DOUBLE(obj); \ - else if (convert_to_double(&(obj), &(dbl)) < 0) \ +#define CONVERT_TO_DOUBLE(obj, dbl) \ + if (PyFloat_Check(obj)) \ + dbl = PyFloat_AS_DOUBLE(obj); \ + else if (_Py_convert_int_to_double(&(obj), &(dbl)) < 0) \ return obj; /* Methods */ -static int -convert_to_double(PyObject **v, double *dbl) +int +_Py_convert_int_to_double(PyObject **v, double *dbl) { PyObject *obj = *v; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 85ebd5b00cc18b..17df9208f224f4 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2829,7 +2829,6 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) double value = PyLong_AsDouble(item); if (value != -1.0 || !PyErr_Occurred()) { re_sum = cs_add(re_sum, value); - im_sum.hi += 0.0; Py_DECREF(item); continue; } @@ -2842,7 +2841,6 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) if (PyFloat_Check(item)) { double value = PyFloat_AS_DOUBLE(item); re_sum = cs_add(re_sum, value); - im_sum.hi += 0.0; _Py_DECREF_SPECIALIZED(item, _PyFloat_ExactDealloc); continue; } From 6da9d252ac39d53342455a17bfec7b1087fba697 Mon Sep 17 00:00:00 2001 From: Sergey Muraviov Date: Tue, 26 Nov 2024 19:39:33 +0300 Subject: [PATCH 028/427] gh-127285: Update PCbuild files for a number of missed changes (GH-127286) --- PCbuild/_elementtree.vcxproj | 1 - PCbuild/_elementtree.vcxproj.filters | 3 -- PCbuild/_testinternalcapi.vcxproj.filters | 3 ++ PCbuild/pcbuild.sln | 62 +++++++++++------------ PCbuild/pylauncher.vcxproj | 2 +- PCbuild/pylauncher.vcxproj.filters | 2 +- PCbuild/python.vcxproj | 2 +- PCbuild/python.vcxproj.filters | 2 +- PCbuild/python_uwp.vcxproj | 2 +- PCbuild/python_uwp.vcxproj.filters | 2 +- PCbuild/pythonw.vcxproj | 3 ++ PCbuild/pythonw.vcxproj.filters | 5 ++ PCbuild/pythonw_uwp.vcxproj | 2 +- PCbuild/pythonw_uwp.vcxproj.filters | 2 +- PCbuild/pywlauncher.vcxproj | 2 +- PCbuild/pywlauncher.vcxproj.filters | 2 +- PCbuild/venvlauncher.vcxproj | 2 +- PCbuild/venvlauncher.vcxproj.filters | 2 +- PCbuild/venvwlauncher.vcxproj | 2 +- PCbuild/venvwlauncher.vcxproj.filters | 2 +- 20 files changed, 56 insertions(+), 49 deletions(-) diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 8c9c0e42f7fe3e..3eb9c89bcb6740 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -106,7 +106,6 @@ - diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index bc14e31f32b95c..a5368024ccebfd 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -42,9 +42,6 @@ Header Files\expat - - Header Files\expat - Header Files\expat diff --git a/PCbuild/_testinternalcapi.vcxproj.filters b/PCbuild/_testinternalcapi.vcxproj.filters index 27429ea5833077..eb0a83ba17f815 100644 --- a/PCbuild/_testinternalcapi.vcxproj.filters +++ b/PCbuild/_testinternalcapi.vcxproj.filters @@ -21,6 +21,9 @@ Source Files + + Source Files + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index d10e1c46a91480..f2a48a7cb63666 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -1099,30 +1099,30 @@ Global {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.Build.0 = Release|Win32 {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.ActiveCfg = Release|x64 {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.Build.0 = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM.ActiveCfg = Debug|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM.Build.0 = Debug|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM64.ActiveCfg = Debug|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM64.Build.0 = Debug|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM.ActiveCfg = Debug|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM.Build.0 = Debug|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM64.Build.0 = Debug|ARM64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.ActiveCfg = Debug|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.Build.0 = Debug|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.ActiveCfg = Debug|x64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.Build.0 = Debug|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM.Build.0 = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM64.ActiveCfg = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM64.Build.0 = Release|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM.ActiveCfg = Release|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM.Build.0 = Release|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM64.ActiveCfg = Release|ARM64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM64.Build.0 = Release|ARM64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.ActiveCfg = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.Build.0 = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|x64.ActiveCfg = Release|x64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|x64.Build.0 = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|ARM.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|ARM64.ActiveCfg = Release|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|ARM.ActiveCfg = Release|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|ARM64.ActiveCfg = Release|ARM64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|Win32.ActiveCfg = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|x64.ActiveCfg = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM.Build.0 = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM64.ActiveCfg = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM64.Build.0 = Release|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM.ActiveCfg = Release|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM.Build.0 = Release|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM64.ActiveCfg = Release|ARM64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM64.Build.0 = Release|ARM64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.ActiveCfg = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.Build.0 = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.ActiveCfg = Release|x64 @@ -1441,26 +1441,26 @@ Global {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.Build.0 = Release|Win32 {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.ActiveCfg = Release|x64 {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.Build.0 = Release|x64 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|ARM.ActiveCfg = Debug|Win32 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|ARM64.ActiveCfg = Debug|Win32 + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|ARM.ActiveCfg = Debug|ARM + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|ARM64.ActiveCfg = Debug|ARM64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|Win32.ActiveCfg = Debug|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|Win32.Build.0 = Debug|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|x64.ActiveCfg = Debug|x64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|x64.Build.0 = Debug|x64 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|ARM.ActiveCfg = PGInstrument|Win32 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|ARM64.ActiveCfg = PGInstrument|Win32 + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|ARM.ActiveCfg = PGUpdate|Win32 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|ARM64.ActiveCfg = PGUpdate|Win32 + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|ARM.ActiveCfg = Release|Win32 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|ARM64.ActiveCfg = Release|Win32 + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|ARM.ActiveCfg = Release|ARM + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|ARM64.ActiveCfg = Release|ARM64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|Win32.ActiveCfg = Release|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|Win32.Build.0 = Release|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|x64.ActiveCfg = Release|x64 @@ -1529,27 +1529,27 @@ Global {FDB84CBB-2FB6-47C8-A2D6-091E0833239D}.Release|Win32.Build.0 = Release|Win32 {FDB84CBB-2FB6-47C8-A2D6-091E0833239D}.Release|x64.ActiveCfg = Release|x64 {FDB84CBB-2FB6-47C8-A2D6-091E0833239D}.Release|x64.Build.0 = Release|x64 - {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM.ActiveCfg = Debug|Win32 - {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM.Build.0 = Debug|Win32 - {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM64.ActiveCfg = Debug|Win32 + {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM.ActiveCfg = Debug|ARM + {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM.Build.0 = Debug|ARM + {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM64.ActiveCfg = Debug|ARM64 {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|Win32.ActiveCfg = Debug|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|Win32.Build.0 = Debug|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|x64.ActiveCfg = Debug|x64 {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|x64.Build.0 = Debug|x64 - {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|ARM.ActiveCfg = PGInstrument|Win32 - {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|ARM64.ActiveCfg = PGInstrument|Win32 + {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|ARM.ActiveCfg = PGUpdate|Win32 - {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|ARM64.ActiveCfg = PGUpdate|Win32 + {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {AB603547-1E2A-45B3-9E09-B04596006393}.Release|ARM.ActiveCfg = Release|Win32 - {AB603547-1E2A-45B3-9E09-B04596006393}.Release|ARM64.ActiveCfg = Release|Win32 + {AB603547-1E2A-45B3-9E09-B04596006393}.Release|ARM.ActiveCfg = Release|ARM + {AB603547-1E2A-45B3-9E09-B04596006393}.Release|ARM64.ActiveCfg = Release|ARM64 {AB603547-1E2A-45B3-9E09-B04596006393}.Release|Win32.ActiveCfg = Release|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.Release|Win32.Build.0 = Release|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.Release|x64.ActiveCfg = Release|x64 diff --git a/PCbuild/pylauncher.vcxproj b/PCbuild/pylauncher.vcxproj index 35f2f7e505bf92..ca422ef4d02e93 100644 --- a/PCbuild/pylauncher.vcxproj +++ b/PCbuild/pylauncher.vcxproj @@ -103,7 +103,7 @@ - + diff --git a/PCbuild/pylauncher.vcxproj.filters b/PCbuild/pylauncher.vcxproj.filters index 17d0389ca50f1c..f10f6d5669e283 100644 --- a/PCbuild/pylauncher.vcxproj.filters +++ b/PCbuild/pylauncher.vcxproj.filters @@ -16,7 +16,7 @@ - + Resource Files diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 4a99ffc677c287..70dabaa3c8bc02 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -103,7 +103,7 @@ - + diff --git a/PCbuild/python.vcxproj.filters b/PCbuild/python.vcxproj.filters index 0662a4e7f5f933..31f8ad9b77058e 100644 --- a/PCbuild/python.vcxproj.filters +++ b/PCbuild/python.vcxproj.filters @@ -9,7 +9,7 @@ - + Resource Files diff --git a/PCbuild/python_uwp.vcxproj b/PCbuild/python_uwp.vcxproj index fb27e9e71222e3..3f8772d30b4ee4 100644 --- a/PCbuild/python_uwp.vcxproj +++ b/PCbuild/python_uwp.vcxproj @@ -108,7 +108,7 @@ - + diff --git a/PCbuild/python_uwp.vcxproj.filters b/PCbuild/python_uwp.vcxproj.filters index 79e87461eb1b7c..76b7a5f29f0a5b 100644 --- a/PCbuild/python_uwp.vcxproj.filters +++ b/PCbuild/python_uwp.vcxproj.filters @@ -9,7 +9,7 @@ - + Resource Files diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index d08c210ef8a1dc..c6a5b8ce90a0d9 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -97,6 +97,9 @@ 3000000 + + + diff --git a/PCbuild/pythonw.vcxproj.filters b/PCbuild/pythonw.vcxproj.filters index 20d87051e2511d..9f92485d1a74d2 100644 --- a/PCbuild/pythonw.vcxproj.filters +++ b/PCbuild/pythonw.vcxproj.filters @@ -18,4 +18,9 @@ Resource Files + + + Resource Files + + \ No newline at end of file diff --git a/PCbuild/pythonw_uwp.vcxproj b/PCbuild/pythonw_uwp.vcxproj index e21e46a1b722ed..f548d022642890 100644 --- a/PCbuild/pythonw_uwp.vcxproj +++ b/PCbuild/pythonw_uwp.vcxproj @@ -108,7 +108,7 @@ - + diff --git a/PCbuild/pythonw_uwp.vcxproj.filters b/PCbuild/pythonw_uwp.vcxproj.filters index 2f39bdea9e910d..72756e14cdc4a9 100644 --- a/PCbuild/pythonw_uwp.vcxproj.filters +++ b/PCbuild/pythonw_uwp.vcxproj.filters @@ -9,7 +9,7 @@ - + Resource Files diff --git a/PCbuild/pywlauncher.vcxproj b/PCbuild/pywlauncher.vcxproj index e50b69aefe2b9c..1694548935a165 100644 --- a/PCbuild/pywlauncher.vcxproj +++ b/PCbuild/pywlauncher.vcxproj @@ -103,7 +103,7 @@ - + diff --git a/PCbuild/pywlauncher.vcxproj.filters b/PCbuild/pywlauncher.vcxproj.filters index 17d0389ca50f1c..f10f6d5669e283 100644 --- a/PCbuild/pywlauncher.vcxproj.filters +++ b/PCbuild/pywlauncher.vcxproj.filters @@ -16,7 +16,7 @@ - + Resource Files diff --git a/PCbuild/venvlauncher.vcxproj b/PCbuild/venvlauncher.vcxproj index 1193e032245c94..abaf3a979af268 100644 --- a/PCbuild/venvlauncher.vcxproj +++ b/PCbuild/venvlauncher.vcxproj @@ -107,7 +107,7 @@ - + diff --git a/PCbuild/venvlauncher.vcxproj.filters b/PCbuild/venvlauncher.vcxproj.filters index 56a0f005a3fa2a..bc98687a6da0d3 100644 --- a/PCbuild/venvlauncher.vcxproj.filters +++ b/PCbuild/venvlauncher.vcxproj.filters @@ -9,7 +9,7 @@ - + Resource Files diff --git a/PCbuild/venvwlauncher.vcxproj b/PCbuild/venvwlauncher.vcxproj index db7128272f06db..c58280deb8abeb 100644 --- a/PCbuild/venvwlauncher.vcxproj +++ b/PCbuild/venvwlauncher.vcxproj @@ -107,7 +107,7 @@ - + diff --git a/PCbuild/venvwlauncher.vcxproj.filters b/PCbuild/venvwlauncher.vcxproj.filters index 61a514395e82dc..5193c38b12d53d 100644 --- a/PCbuild/venvwlauncher.vcxproj.filters +++ b/PCbuild/venvwlauncher.vcxproj.filters @@ -14,7 +14,7 @@ - + Resource Files From f0d3f10c43c9029378adba11a65b3d1287e4be32 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 26 Nov 2024 21:20:37 +0300 Subject: [PATCH 029/427] gh-69639: Add mixed-mode rules for complex arithmetic (C-like) (GH-124829) "Generally, mixed-mode arithmetic combining real and complex variables should be performed directly, not by first coercing the real to complex, lest the sign of zero be rendered uninformative; the same goes for combinations of pure imaginary quantities with complex variables." (c) Kahan, W: Branch cuts for complex elementary functions. This patch implements mixed-mode arithmetic rules, combining real and complex variables as specified by C standards since C99 (in particular, there is no special version for the true division with real lhs operand). Most C compilers implementing C99+ Annex G have only these special rules (without support for imaginary type, which is going to be deprecated in C2y). From 71ede1142ddad2d31cc966b8fe4a5aff664f4d53 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 26 Nov 2024 21:46:06 +0000 Subject: [PATCH 030/427] gh-115999: Add free-threaded specialization for `STORE_SUBSCR` (#127169) The specialization only depends on the type, so no special thread-safety considerations there. STORE_SUBSCR_LIST_INT needs to lock the list before modifying it. `_PyDict_SetItem_Take2` already internally locks the dictionary using a critical section. --- Python/bytecodes.c | 11 +++- Python/ceval_macros.h | 23 +++++++ Python/executor_cases.c.h | 12 +++- Python/generated_cases.c.h | 11 +++- Python/specialize.c | 122 ++++++++++++++++++------------------- 5 files changed, 109 insertions(+), 70 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 88e96afe4151f5..a14b32b8108be8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -910,7 +910,7 @@ dummy_func( }; specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); @@ -918,7 +918,7 @@ dummy_func( } OPCODE_DEFERRED_INC(STORE_SUBSCR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_STORE_SUBSCR, (v, container, sub -- )) { @@ -940,13 +940,18 @@ dummy_func( // Ensure nonnegative, zero-or-one-digit ints. DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(!LOCK_OBJECT(list)); // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list)); + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + DEOPT_IF(true); + } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! Py_DECREF(old_value); PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); DEAD(sub_st); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 603b71ea938cde..9250b86e42ced1 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -284,6 +284,29 @@ GETITEM(PyObject *v, Py_ssize_t i) { } +// Try to lock an object in the free threading build, if it's not already +// locked. Use with a DEOPT_IF() to deopt if the object is already locked. +// These are no-ops in the default GIL build. The general pattern is: +// +// DEOPT_IF(!LOCK_OBJECT(op)); +// if (/* condition fails */) { +// UNLOCK_OBJECT(op); +// DEOPT_IF(true); +// } +// ... +// UNLOCK_OBJECT(op); +// +// NOTE: The object must be unlocked on every exit code path and you should +// avoid any potentially escaping calls (like PyStackRef_CLOSE) while the +// object is locked. +#ifdef Py_GIL_DISABLED +# define LOCK_OBJECT(op) PyMutex_LockFast(&(_PyObject_CAST(op))->ob_mutex._bits) +# define UNLOCK_OBJECT(op) PyMutex_Unlock(&(_PyObject_CAST(op))->ob_mutex) +#else +# define LOCK_OBJECT(op) (1) +# define UNLOCK_OBJECT(op) ((void)0) +#endif + #define GLOBALS() frame->f_globals #define BUILTINS() frame->f_builtins #define LOCALS() frame->f_locals diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 5af970ec4ae219..d46412a193332b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1235,15 +1235,23 @@ JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) { + if (!LOCK_OBJECT(list)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + // Ensure index < len(list) + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! Py_DECREF(old_value); PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(list_st); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 36ec727eed3fc9..c9a5132269398c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7629,7 +7629,7 @@ container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7639,7 +7639,7 @@ } OPCODE_DEFERRED_INC(STORE_SUBSCR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _STORE_SUBSCR { @@ -7704,12 +7704,17 @@ // Ensure nonnegative, zero-or-one-digit ints. DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + DEOPT_IF(true, STORE_SUBSCR); + } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! Py_DECREF(old_value); PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(list_st); diff --git a/Python/specialize.c b/Python/specialize.c index c1f4b0601cc8d5..172dae7d374602 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1814,86 +1814,54 @@ _Py_Specialize_BinarySubscr( cache->counter = adaptive_counter_cooldown(); } -void -_Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) -{ - PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - assert(ENABLE_SPECIALIZATION); - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1); - PyTypeObject *container_type = Py_TYPE(container); - if (container_type == &PyList_Type) { - if (PyLong_CheckExact(sub)) { - if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub) - && ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container)) - { - instr->op.code = STORE_SUBSCR_LIST_INT; - goto success; - } - else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); - goto fail; - } - } - else if (PySlice_Check(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE); - goto fail; - } - else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); - goto fail; - } - } - if (container_type == &PyDict_Type) { - instr->op.code = STORE_SUBSCR_DICT; - goto success; - } #ifdef Py_STATS +static int +store_subscr_fail_kind(PyObject *container_type) +{ PyMappingMethods *as_mapping = container_type->tp_as_mapping; if (as_mapping && (as_mapping->mp_ass_subscript == PyDict_Type.tp_as_mapping->mp_ass_subscript)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE); - goto fail; + return SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE; } if (PyObject_CheckBuffer(container)) { if (PyLong_CheckExact(sub) && (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub))) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + return SPEC_FAIL_OUT_OF_RANGE; } else if (strcmp(container_type->tp_name, "array.array") == 0) { if (PyLong_CheckExact(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_INT); + return SPEC_FAIL_SUBSCR_ARRAY_INT; } else if (PySlice_Check(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_SLICE); + return SPEC_FAIL_SUBSCR_ARRAY_SLICE; } else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); + return SPEC_FAIL_OTHER; } } else if (PyByteArray_CheckExact(container)) { if (PyLong_CheckExact(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_INT); + return SPEC_FAIL_SUBSCR_BYTEARRAY_INT; } else if (PySlice_Check(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE); + return SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE; } else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); + return SPEC_FAIL_OTHER; } } else { if (PyLong_CheckExact(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_INT); + return SPEC_FAIL_SUBSCR_BUFFER_INT; } else if (PySlice_Check(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_SLICE); + return SPEC_FAIL_SUBSCR_BUFFER_SLICE; } else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); + return SPEC_FAIL_OTHER; } } - goto fail; + return SPEC_FAIL_OTHER; } PyObject *descriptor = _PyType_Lookup(container_type, &_Py_ID(__setitem__)); if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { @@ -1901,25 +1869,55 @@ _Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_COD PyCodeObject *code = (PyCodeObject *)func->func_code; int kind = function_kind(code); if (kind == SIMPLE_FUNCTION) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_SIMPLE); + return SPEC_FAIL_SUBSCR_PY_SIMPLE; } else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_OTHER); + return SPEC_FAIL_SUBSCR_PY_OTHER; } - goto fail; } -#endif // Py_STATS - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); -fail: - STAT_INC(STORE_SUBSCR, failure); - assert(!PyErr_Occurred()); - instr->op.code = STORE_SUBSCR; - cache->counter = adaptive_counter_backoff(cache->counter); - return; -success: - STAT_INC(STORE_SUBSCR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + return SPEC_FAIL_OTHER; +} +#endif + +void +_Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) +{ + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + + assert(ENABLE_SPECIALIZATION_FT); + PyTypeObject *container_type = Py_TYPE(container); + if (container_type == &PyList_Type) { + if (PyLong_CheckExact(sub)) { + if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub) + && ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container)) + { + specialize(instr, STORE_SUBSCR_LIST_INT); + return; + } + else { + SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + unspecialize(instr); + return; + } + } + else if (PySlice_Check(sub)) { + SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE); + unspecialize(instr); + return; + } + else { + SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); + unspecialize(instr); + return; + } + } + if (container_type == &PyDict_Type) { + specialize(instr, STORE_SUBSCR_DICT); + return; + } + SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container_type)); + unspecialize(instr); } /* Returns a borrowed reference. From 6d3b5206cfaf5a85c128b671b1d9527ed553c930 Mon Sep 17 00:00:00 2001 From: "RUANG (James Roy)" Date: Wed, 27 Nov 2024 14:53:41 +0800 Subject: [PATCH 031/427] gh-127072: Remove outdated `socket.NETLINK_*` constants. (GH-127256) Remove seriously outdated netlink constants. Co-authored-by: Gregory P. Smith --- ...-11-25-19-04-10.gh-issue-127072.-c284K.rst | 1 + Modules/socketmodule.c | 20 ++----------------- 2 files changed, 3 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-25-19-04-10.gh-issue-127072.-c284K.rst diff --git a/Misc/NEWS.d/next/Library/2024-11-25-19-04-10.gh-issue-127072.-c284K.rst b/Misc/NEWS.d/next/Library/2024-11-25-19-04-10.gh-issue-127072.-c284K.rst new file mode 100644 index 00000000000000..1bc7e1f0de9e0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-25-19-04-10.gh-issue-127072.-c284K.rst @@ -0,0 +1 @@ +Remove outdated ``socket.NETLINK_*`` constants not present in Linux kernels beyond 2.6.17. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 06be822c9555f9..9394f1c940bedf 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7594,36 +7594,20 @@ socket_exec(PyObject *m) /* */ ADD_INT_MACRO(m, AF_NETLINK); ADD_INT_MACRO(m, NETLINK_ROUTE); -#ifdef NETLINK_SKIP - ADD_INT_MACRO(m, NETLINK_SKIP); -#endif -#ifdef NETLINK_W1 - ADD_INT_MACRO(m, NETLINK_W1); -#endif ADD_INT_MACRO(m, NETLINK_USERSOCK); ADD_INT_MACRO(m, NETLINK_FIREWALL); -#ifdef NETLINK_TCPDIAG - ADD_INT_MACRO(m, NETLINK_TCPDIAG); -#endif #ifdef NETLINK_NFLOG ADD_INT_MACRO(m, NETLINK_NFLOG); #endif #ifdef NETLINK_XFRM ADD_INT_MACRO(m, NETLINK_XFRM); #endif -#ifdef NETLINK_ARPD - ADD_INT_MACRO(m, NETLINK_ARPD); -#endif -#ifdef NETLINK_ROUTE6 - ADD_INT_MACRO(m, NETLINK_ROUTE6); -#endif +#ifdef NETLINK_IP6_FW ADD_INT_MACRO(m, NETLINK_IP6_FW); +#endif #ifdef NETLINK_DNRTMSG ADD_INT_MACRO(m, NETLINK_DNRTMSG); #endif -#ifdef NETLINK_TAPBASE - ADD_INT_MACRO(m, NETLINK_TAPBASE); -#endif #ifdef NETLINK_CRYPTO ADD_INT_MACRO(m, NETLINK_CRYPTO); #endif From 83926d3b4c7847394b5e2531e9566d7fc9fbea0f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 27 Nov 2024 13:29:50 +0200 Subject: [PATCH 032/427] gh-109746: Fix race condition in test_start_new_thread_failed (GH-127299) --- Lib/test/test_threading.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index fb6d268e5869f4..b666533466e578 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1196,11 +1196,11 @@ def f(): except RuntimeError: print('ok') else: - print('skip') + print('!skip!') """ _, out, err = assert_python_ok("-u", "-c", code) out = out.strip() - if out == b'skip': + if b'!skip!' in out: self.skipTest('RLIMIT_NPROC had no effect; probably superuser') self.assertEqual(out, b'ok') self.assertEqual(err, b'') From 3cf83d91a5baf3600dd60f7aaaf4fb6d73c4b8a9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 27 Nov 2024 13:38:12 +0200 Subject: [PATCH 033/427] gh-124008: Fix calculation of the number of written bytes for the Windows console (GH-124059) Since MultiByteToWideChar()/WideCharToMultiByte() is not reversible if the data contains invalid UTF-8 sequences, use binary search to calculate the number of written bytes from the number of written characters. Also fix writing incomplete UTF-8 sequences. Also fix handling of memory allocation failures. --- Lib/test/test_winconsoleio.py | 23 ++++ ...-09-13-18-24-27.gh-issue-124008.XaiPQx.rst | 2 + Modules/_io/winconsoleio.c | 118 +++++++++++++----- 3 files changed, 115 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-13-18-24-27.gh-issue-124008.XaiPQx.rst diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index a10d63dfdc9753..d9076e77c158a2 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -142,6 +142,29 @@ def test_write_empty_data(self): with ConIO('CONOUT$', 'w') as f: self.assertEqual(f.write(b''), 0) + @requires_resource('console') + def test_write(self): + testcases = [] + with ConIO('CONOUT$', 'w') as f: + for a in [ + b'', + b'abc', + b'\xc2\xa7\xe2\x98\x83\xf0\x9f\x90\x8d', + b'\xff'*10, + ]: + for b in b'\xc2\xa7', b'\xe2\x98\x83', b'\xf0\x9f\x90\x8d': + testcases.append(a + b) + for i in range(1, len(b)): + data = a + b[:i] + testcases.append(data + b'z') + testcases.append(data + b'\xff') + # incomplete multibyte sequence + with self.subTest(data=data): + self.assertEqual(f.write(data), len(a)) + for data in testcases: + with self.subTest(data=data): + self.assertEqual(f.write(data), len(data)) + def assertStdinRoundTrip(self, text): stdin = open('CONIN$', 'r') old_stdin = sys.stdin diff --git a/Misc/NEWS.d/next/Library/2024-09-13-18-24-27.gh-issue-124008.XaiPQx.rst b/Misc/NEWS.d/next/Library/2024-09-13-18-24-27.gh-issue-124008.XaiPQx.rst new file mode 100644 index 00000000000000..cd6dd9a7a97e90 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-13-18-24-27.gh-issue-124008.XaiPQx.rst @@ -0,0 +1,2 @@ +Fix possible crash (in debug build), incorrect output or returning incorrect +value from raw binary ``write()`` when writing to console on Windows. diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index d7cb5abfdc0abd..3fa0301e337991 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -135,19 +135,67 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { } static DWORD -_find_last_utf8_boundary(const char *buf, DWORD len) +_find_last_utf8_boundary(const unsigned char *buf, DWORD len) { - /* This function never returns 0, returns the original len instead */ - DWORD count = 1; - if (len == 0 || (buf[len - 1] & 0x80) == 0) { - return len; - } - for (;; count++) { - if (count > 3 || count >= len) { + for (DWORD count = 1; count < 4 && count <= len; count++) { + unsigned char c = buf[len - count]; + if (c < 0x80) { + /* No starting byte found. */ return len; } - if ((buf[len - count] & 0xc0) != 0x80) { - return len - count; + if (c >= 0xc0) { + if (c < 0xe0 /* 2-bytes sequence */ ? count < 2 : + c < 0xf0 /* 3-bytes sequence */ ? count < 3 : + c < 0xf8 /* 4-bytes sequence */) + { + /* Incomplete multibyte sequence. */ + return len - count; + } + /* Either complete or invalid sequence. */ + return len; + } + } + /* Either complete 4-bytes sequence or invalid sequence. */ + return len; +} + +/* Find the number of UTF-8 bytes that corresponds to the specified number of + * wchars. + * I.e. find x <= len so that MultiByteToWideChar(CP_UTF8, 0, s, x, NULL, 0) == n. + * + * WideCharToMultiByte() cannot be used for this, because the UTF-8 -> wchar + * conversion is not reversible (invalid UTF-8 byte produces \ufffd which + * will be converted back to 3-bytes UTF-8 sequence \xef\xbf\xbd). + * So we need to use binary search. + */ +static DWORD +_wchar_to_utf8_count(const unsigned char *s, DWORD len, DWORD n) +{ + DWORD start = 0; + while (1) { + DWORD mid = 0; + for (DWORD i = len / 2; i <= len; i++) { + mid = _find_last_utf8_boundary(s, i); + if (mid != 0) { + break; + } + /* The middle could split the first multibytes sequence. */ + } + if (mid == len) { + return start + len; + } + if (mid == 0) { + mid = len > 1 ? len - 1 : 1; + } + DWORD wlen = MultiByteToWideChar(CP_UTF8, 0, s, mid, NULL, 0); + if (wlen <= n) { + s += mid; + start += mid; + len -= mid; + n -= wlen; + } + else { + len = mid; } } } @@ -563,8 +611,10 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { int err = 0, sig = 0; wchar_t *buf = (wchar_t*)PyMem_Malloc(maxlen * sizeof(wchar_t)); - if (!buf) + if (!buf) { + PyErr_NoMemory(); goto error; + } *readlen = 0; @@ -622,6 +672,7 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { Py_UNBLOCK_THREADS if (!newbuf) { sig = -1; + PyErr_NoMemory(); break; } buf = newbuf; @@ -645,8 +696,10 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { if (*readlen > 0 && buf[0] == L'\x1a') { PyMem_Free(buf); buf = (wchar_t *)PyMem_Malloc(sizeof(wchar_t)); - if (!buf) + if (!buf) { + PyErr_NoMemory(); goto error; + } buf[0] = L'\0'; *readlen = 0; } @@ -824,8 +877,10 @@ _io__WindowsConsoleIO_readall_impl(winconsoleio *self) bufsize = BUFSIZ; buf = (wchar_t*)PyMem_Malloc((bufsize + 1) * sizeof(wchar_t)); - if (buf == NULL) + if (buf == NULL) { + PyErr_NoMemory(); return NULL; + } while (1) { wchar_t *subbuf; @@ -847,6 +902,7 @@ _io__WindowsConsoleIO_readall_impl(winconsoleio *self) (bufsize + 1) * sizeof(wchar_t)); if (tmp == NULL) { PyMem_Free(buf); + PyErr_NoMemory(); return NULL; } buf = tmp; @@ -1022,43 +1078,49 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, len = (DWORD)b->len; Py_BEGIN_ALLOW_THREADS - wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); - /* issue11395 there is an unspecified upper bound on how many bytes can be written at once. We cap at 32k - the caller will have to handle partial writes. Since we don't know how many input bytes are being ignored, we have to reduce and recalculate. */ - while (wlen > 32766 / sizeof(wchar_t)) { - len /= 2; + const DWORD max_wlen = 32766 / sizeof(wchar_t); + /* UTF-8 to wchar ratio is at most 3:1. */ + len = Py_MIN(len, max_wlen * 3); + while (1) { /* Fix for github issues gh-110913 and gh-82052. */ len = _find_last_utf8_boundary(b->buf, len); wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); + if (wlen <= max_wlen) { + break; + } + len /= 2; } Py_END_ALLOW_THREADS - if (!wlen) - return PyErr_SetFromWindowsErr(0); + if (!wlen) { + return PyLong_FromLong(0); + } wbuf = (wchar_t*)PyMem_Malloc(wlen * sizeof(wchar_t)); + if (!wbuf) { + PyErr_NoMemory(); + return NULL; + } Py_BEGIN_ALLOW_THREADS wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, wbuf, wlen); if (wlen) { res = WriteConsoleW(handle, wbuf, wlen, &n, NULL); +#ifdef Py_DEBUG + if (res) { +#else if (res && n < wlen) { +#endif /* Wrote fewer characters than expected, which means our * len value may be wrong. So recalculate it from the - * characters that were written. As this could potentially - * result in a different value, we also validate that value. + * characters that were written. */ - len = WideCharToMultiByte(CP_UTF8, 0, wbuf, n, - NULL, 0, NULL, NULL); - if (len) { - wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, - NULL, 0); - assert(wlen == len); - } + len = _wchar_to_utf8_count(b->buf, len, n); } } else res = 0; From 9328db7652677a23192cb51b0324a0fdbfa587c9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 27 Nov 2024 06:38:55 -0800 Subject: [PATCH 034/427] Fix indentation for contextlib.asynccontextmanager docs (#127333) --- Doc/library/contextlib.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index f5b349441bcfee..e8f264f949807d 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -151,9 +151,9 @@ Functions and classes provided: created by :func:`asynccontextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators. - .. versionchanged:: 3.10 - Async context managers created with :func:`asynccontextmanager` can - be used as decorators. + .. versionchanged:: 3.10 + Async context managers created with :func:`asynccontextmanager` can + be used as decorators. .. function:: closing(thing) From 58e334e1431b2ed6b70ee42501ea73e08084e769 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Nov 2024 16:14:49 +0100 Subject: [PATCH 035/427] gh-123967: Fix faulthandler for trampoline frames (#127329) If the top-most frame is a trampoline frame, skip it. --- ...-11-27-14-06-35.gh-issue-123967.wxUmnW.rst | 2 ++ Python/traceback.c | 23 +++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-27-14-06-35.gh-issue-123967.wxUmnW.rst diff --git a/Misc/NEWS.d/next/Library/2024-11-27-14-06-35.gh-issue-123967.wxUmnW.rst b/Misc/NEWS.d/next/Library/2024-11-27-14-06-35.gh-issue-123967.wxUmnW.rst new file mode 100644 index 00000000000000..788fe0c78ef257 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-27-14-06-35.gh-issue-123967.wxUmnW.rst @@ -0,0 +1,2 @@ +Fix faulthandler for trampoline frames. If the top-most frame is a +trampoline frame, skip it. Patch by Victor Stinner. diff --git a/Python/traceback.c b/Python/traceback.c index 47b77c9108dd9a..e819909b6045c3 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -890,6 +890,8 @@ _Py_DumpASCII(int fd, PyObject *text) static void dump_frame(int fd, _PyInterpreterFrame *frame) { + assert(frame->owner != FRAME_OWNED_BY_CSTACK); + PyCodeObject *code =_PyFrame_GetCode(frame); PUTS(fd, " File "); if (code->co_filename != NULL @@ -963,6 +965,17 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) unsigned int depth = 0; while (1) { + if (frame->owner == FRAME_OWNED_BY_CSTACK) { + /* Trampoline frame */ + frame = frame->previous; + if (frame == NULL) { + break; + } + + /* Can't have more than one shim frame in a row */ + assert(frame->owner != FRAME_OWNED_BY_CSTACK); + } + if (MAX_FRAME_DEPTH <= depth) { if (MAX_FRAME_DEPTH < depth) { PUTS(fd, "plus "); @@ -971,20 +984,12 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) } break; } + dump_frame(fd, frame); frame = frame->previous; if (frame == NULL) { break; } - if (frame->owner == FRAME_OWNED_BY_CSTACK) { - /* Trampoline frame */ - frame = frame->previous; - } - if (frame == NULL) { - break; - } - /* Can't have more than one shim frame in a row */ - assert(frame->owner != FRAME_OWNED_BY_CSTACK); depth++; } } From 14a05a8f433197c40293028f01797da444fb8409 Mon Sep 17 00:00:00 2001 From: Topher Fischer Date: Wed, 27 Nov 2024 07:24:45 -0800 Subject: [PATCH 036/427] gh-126882: Fix indentation in code sample block (#126887) --- InternalDocs/garbage_collector.md | 156 ++++++++++++++---------------- 1 file changed, 75 insertions(+), 81 deletions(-) diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 5de4aa05398b55..272a0834cbfe24 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -17,26 +17,26 @@ value returned by this function is always 1 more as the function also has a refe to the object when called): ```pycon - >>> x = object() - >>> sys.getrefcount(x) - 2 - >>> y = x - >>> sys.getrefcount(x) - 3 - >>> del y - >>> sys.getrefcount(x) - 2 +>>> x = object() +>>> sys.getrefcount(x) +2 +>>> y = x +>>> sys.getrefcount(x) +3 +>>> del y +>>> sys.getrefcount(x) +2 ``` The main problem with the reference counting scheme is that it does not handle reference cycles. For instance, consider this code: ```pycon - >>> container = [] - >>> container.append(container) - >>> sys.getrefcount(container) - 3 - >>> del container +>>> container = [] +>>> container.append(container) +>>> sys.getrefcount(container) +3 +>>> del container ``` In this example, `container` holds a reference to itself, so even when we remove @@ -199,26 +199,26 @@ variable `A`, and one self-referencing object which is completely unreachable: ```pycon - >>> import gc - - >>> class Link: - ... def __init__(self, next_link=None): - ... self.next_link = next_link - - >>> link_3 = Link() - >>> link_2 = Link(link_3) - >>> link_1 = Link(link_2) - >>> link_3.next_link = link_1 - >>> A = link_1 - >>> del link_1, link_2, link_3 - - >>> link_4 = Link() - >>> link_4.next_link = link_4 - >>> del link_4 - - # Collect the unreachable Link object (and its .__dict__ dict). - >>> gc.collect() - 2 +>>> import gc +>>> +>>> class Link: +... def __init__(self, next_link=None): +... self.next_link = next_link +... +>>> link_3 = Link() +>>> link_2 = Link(link_3) +>>> link_1 = Link(link_2) +>>> link_3.next_link = link_1 +>>> A = link_1 +>>> del link_1, link_2, link_3 +>>> +>>> link_4 = Link() +>>> link_4.next_link = link_4 +>>> del link_4 +>>> +>>> # Collect the unreachable Link object (and its .__dict__ dict). +>>> gc.collect() +2 ``` The GC starts with a set of candidate objects it wants to scan. In the @@ -439,9 +439,9 @@ These thresholds can be examined using the function: ```pycon - >>> import gc - >>> gc.get_threshold() - (700, 10, 10) +>>> import gc +>>> gc.get_threshold() +(700, 10, 10) ``` The content of these generations can be examined using the @@ -449,38 +449,32 @@ The content of these generations can be examined using the specifically in a generation by calling `gc.collect(generation=NUM)`. ```pycon - >>> import gc - >>> class MyObj: - ... pass - ... - - # Move everything to the old generation so it's easier to inspect - # the young generation. - - >>> gc.collect() - 0 - - # Create a reference cycle. - - >>> x = MyObj() - >>> x.self = x - - # Initially the object is in the young generation. - - >>> gc.get_objects(generation=0) - [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] - - # After a collection of the youngest generation the object - # moves to the old generation. - - >>> gc.collect(generation=0) - 0 - >>> gc.get_objects(generation=0) - [] - >>> gc.get_objects(generation=1) - [] - >>> gc.get_objects(generation=2) - [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] +>>> import gc +>>> class MyObj: +... pass +... +>>> # Move everything to the old generation so it's easier to inspect +>>> # the young generation. +>>> gc.collect() +0 +>>> # Create a reference cycle. +>>> x = MyObj() +>>> x.self = x +>>> +>>> # Initially the object is in the young generation. +>>> gc.get_objects(generation=0) +[..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] +>>> +>>> # After a collection of the youngest generation the object +>>> # moves to the old generation. +>>> gc.collect(generation=0) +0 +>>> gc.get_objects(generation=0) +[] +>>> gc.get_objects(generation=1) +[] +>>> gc.get_objects(generation=2) +[..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ``` @@ -563,18 +557,18 @@ the current tracking status of the object. Subsequent garbage collections may ch tracking status of the object. ```pycon - >>> gc.is_tracked(0) - False - >>> gc.is_tracked("a") - False - >>> gc.is_tracked([]) - True - >>> gc.is_tracked(()) - False - >>> gc.is_tracked({}) - True - >>> gc.is_tracked({"a": 1}) - True +>>> gc.is_tracked(0) +False +>>> gc.is_tracked("a") +False +>>> gc.is_tracked([]) +True +>>> gc.is_tracked(()) +False +>>> gc.is_tracked({}) +True +>>> gc.is_tracked({"a": 1}) +True ``` Differences between GC implementations From 78cb377c622a98b1bf58df40c828e886575a6927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 27 Nov 2024 17:42:45 +0100 Subject: [PATCH 037/427] gh-122288: Improve performances of `fnmatch.translate` (#122289) Improve performance of this function by a factor of 1.7x. Co-authored-by: Barney Gale --- Lib/fnmatch.py | 90 +++++++++---------- Lib/glob.py | 2 +- Lib/test/test_fnmatch.py | 69 ++++++++++++++ ...-07-25-18-06-51.gh-issue-122288.-_xxOR.rst | 2 + 4 files changed, 114 insertions(+), 49 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-07-25-18-06-51.gh-issue-122288.-_xxOR.rst diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index 73acb1fe8d4106..865baea23467ea 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -77,24 +77,30 @@ def translate(pat): There is no way to quote meta-characters. """ - STAR = object() - parts = _translate(pat, STAR, '.') - return _join_translated_parts(parts, STAR) + parts, star_indices = _translate(pat, '*', '.') + return _join_translated_parts(parts, star_indices) +_re_setops_sub = re.compile(r'([&~|])').sub +_re_escape = functools.lru_cache(maxsize=512)(re.escape) -def _translate(pat, STAR, QUESTION_MARK): +def _translate(pat, star, question_mark): res = [] add = res.append + star_indices = [] + i, n = 0, len(pat) while i < n: c = pat[i] i = i+1 if c == '*': + # store the position of the wildcard + star_indices.append(len(res)) + add(star) # compress consecutive `*` into one - if (not res) or res[-1] is not STAR: - add(STAR) + while i < n and pat[i] == '*': + i += 1 elif c == '?': - add(QUESTION_MARK) + add(question_mark) elif c == '[': j = i if j < n and pat[j] == '!': @@ -133,8 +139,6 @@ def _translate(pat, STAR, QUESTION_MARK): # Hyphens that create ranges shouldn't be escaped. stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') for s in chunks) - # Escape set operations (&&, ~~ and ||). - stuff = re.sub(r'([&~|])', r'\\\1', stuff) i = j+1 if not stuff: # Empty range: never match. @@ -143,50 +147,40 @@ def _translate(pat, STAR, QUESTION_MARK): # Negated empty range: match any character. add('.') else: + # Escape set operations (&&, ~~ and ||). + stuff = _re_setops_sub(r'\\\1', stuff) if stuff[0] == '!': stuff = '^' + stuff[1:] elif stuff[0] in ('^', '['): stuff = '\\' + stuff add(f'[{stuff}]') else: - add(re.escape(c)) - assert i == n - return res - - -def _join_translated_parts(inp, STAR): - # Deal with STARs. - res = [] - add = res.append - i, n = 0, len(inp) - # Fixed pieces at the start? - while i < n and inp[i] is not STAR: - add(inp[i]) - i += 1 - # Now deal with STAR fixed STAR fixed ... - # For an interior `STAR fixed` pairing, we want to do a minimal - # .*? match followed by `fixed`, with no possibility of backtracking. - # Atomic groups ("(?>...)") allow us to spell that directly. - # Note: people rely on the undocumented ability to join multiple - # translate() results together via "|" to build large regexps matching - # "one of many" shell patterns. - while i < n: - assert inp[i] is STAR - i += 1 - if i == n: - add(".*") - break - assert inp[i] is not STAR - fixed = [] - while i < n and inp[i] is not STAR: - fixed.append(inp[i]) - i += 1 - fixed = "".join(fixed) - if i == n: - add(".*") - add(fixed) - else: - add(f"(?>.*?{fixed})") + add(_re_escape(c)) assert i == n - res = "".join(res) + return res, star_indices + + +def _join_translated_parts(parts, star_indices): + if not star_indices: + return fr'(?s:{"".join(parts)})\Z' + iter_star_indices = iter(star_indices) + j = next(iter_star_indices) + buffer = parts[:j] # fixed pieces at the start + append, extend = buffer.append, buffer.extend + i = j + 1 + for j in iter_star_indices: + # Now deal with STAR fixed STAR fixed ... + # For an interior `STAR fixed` pairing, we want to do a minimal + # .*? match followed by `fixed`, with no possibility of backtracking. + # Atomic groups ("(?>...)") allow us to spell that directly. + # Note: people rely on the undocumented ability to join multiple + # translate() results together via "|" to build large regexps matching + # "one of many" shell patterns. + append('(?>.*?') + extend(parts[i:j]) + append(')') + i = j + 1 + append('.*') + extend(parts[i:]) + res = ''.join(buffer) return fr'(?s:{res})\Z' diff --git a/Lib/glob.py b/Lib/glob.py index ce9b3698888dd9..690ab1b8b9fb1d 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -312,7 +312,7 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None): if part: if not include_hidden and part[0] in '*?': results.append(r'(?!\.)') - results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)[0]) if idx < last_part_idx: results.append(any_sep) res = ''.join(results) diff --git a/Lib/test/test_fnmatch.py b/Lib/test/test_fnmatch.py index 10ed496d4e2f37..9f360e1dc10f47 100644 --- a/Lib/test/test_fnmatch.py +++ b/Lib/test/test_fnmatch.py @@ -250,6 +250,75 @@ def test_translate(self): self.assertTrue(re.match(fatre, 'cbabcaxc')) self.assertFalse(re.match(fatre, 'dabccbad')) + def test_translate_wildcards(self): + for pattern, expect in [ + ('ab*', r'(?s:ab.*)\Z'), + ('ab*cd', r'(?s:ab.*cd)\Z'), + ('ab*cd*', r'(?s:ab(?>.*?cd).*)\Z'), + ('ab*cd*12', r'(?s:ab(?>.*?cd).*12)\Z'), + ('ab*cd*12*', r'(?s:ab(?>.*?cd)(?>.*?12).*)\Z'), + ('ab*cd*12*34', r'(?s:ab(?>.*?cd)(?>.*?12).*34)\Z'), + ('ab*cd*12*34*', r'(?s:ab(?>.*?cd)(?>.*?12)(?>.*?34).*)\Z'), + ]: + with self.subTest(pattern): + translated = translate(pattern) + self.assertEqual(translated, expect, pattern) + + for pattern, expect in [ + ('*ab', r'(?s:.*ab)\Z'), + ('*ab*', r'(?s:(?>.*?ab).*)\Z'), + ('*ab*cd', r'(?s:(?>.*?ab).*cd)\Z'), + ('*ab*cd*', r'(?s:(?>.*?ab)(?>.*?cd).*)\Z'), + ('*ab*cd*12', r'(?s:(?>.*?ab)(?>.*?cd).*12)\Z'), + ('*ab*cd*12*', r'(?s:(?>.*?ab)(?>.*?cd)(?>.*?12).*)\Z'), + ('*ab*cd*12*34', r'(?s:(?>.*?ab)(?>.*?cd)(?>.*?12).*34)\Z'), + ('*ab*cd*12*34*', r'(?s:(?>.*?ab)(?>.*?cd)(?>.*?12)(?>.*?34).*)\Z'), + ]: + with self.subTest(pattern): + translated = translate(pattern) + self.assertEqual(translated, expect, pattern) + + def test_translate_expressions(self): + for pattern, expect in [ + ('[', r'(?s:\[)\Z'), + ('[!', r'(?s:\[!)\Z'), + ('[]', r'(?s:\[\])\Z'), + ('[abc', r'(?s:\[abc)\Z'), + ('[!abc', r'(?s:\[!abc)\Z'), + ('[abc]', r'(?s:[abc])\Z'), + ('[!abc]', r'(?s:[^abc])\Z'), + ('[!abc][!def]', r'(?s:[^abc][^def])\Z'), + # with [[ + ('[[', r'(?s:\[\[)\Z'), + ('[[a', r'(?s:\[\[a)\Z'), + ('[[]', r'(?s:[\[])\Z'), + ('[[]a', r'(?s:[\[]a)\Z'), + ('[[]]', r'(?s:[\[]\])\Z'), + ('[[]a]', r'(?s:[\[]a\])\Z'), + ('[[a]', r'(?s:[\[a])\Z'), + ('[[a]]', r'(?s:[\[a]\])\Z'), + ('[[a]b', r'(?s:[\[a]b)\Z'), + # backslashes + ('[\\', r'(?s:\[\\)\Z'), + (r'[\]', r'(?s:[\\])\Z'), + (r'[\\]', r'(?s:[\\\\])\Z'), + ]: + with self.subTest(pattern): + translated = translate(pattern) + self.assertEqual(translated, expect, pattern) + + def test_star_indices_locations(self): + from fnmatch import _translate + + blocks = ['a^b', '***', '?', '?', '[a-z]', '[1-9]', '*', '++', '[[a'] + parts, star_indices = _translate(''.join(blocks), '*', '.') + expect_parts = ['a', r'\^', 'b', '*', + '.', '.', '[a-z]', '[1-9]', '*', + r'\+', r'\+', r'\[', r'\[', 'a'] + self.assertListEqual(parts, expect_parts) + self.assertListEqual(star_indices, [3, 8]) + + class FilterTestCase(unittest.TestCase): def test_filter(self): diff --git a/Misc/NEWS.d/next/Library/2024-07-25-18-06-51.gh-issue-122288.-_xxOR.rst b/Misc/NEWS.d/next/Library/2024-07-25-18-06-51.gh-issue-122288.-_xxOR.rst new file mode 100644 index 00000000000000..26a18afca945d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-25-18-06-51.gh-issue-122288.-_xxOR.rst @@ -0,0 +1,2 @@ +Improve the performances of :func:`fnmatch.translate` by a factor 1.7. Patch +by Bénédikt Tran. From 2247dd0f11058502f44b289df0e18ecc08be8657 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 27 Nov 2024 14:36:36 -0800 Subject: [PATCH 038/427] GH-127154: Remove PGO and unknown-linux-gnu/clang from JIT CI (#127212) --- .github/workflows/jit.yml | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 35d5d59b762660..7dbbe71b2131e7 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -54,9 +54,7 @@ jobs: - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc - - x86_64-unknown-linux-gnu/clang - aarch64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/clang debug: - true - false @@ -66,41 +64,24 @@ jobs: - target: i686-pc-windows-msvc/msvc architecture: Win32 runner: windows-latest - compiler: msvc - target: x86_64-pc-windows-msvc/msvc architecture: x64 runner: windows-latest - compiler: msvc - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-latest - compiler: msvc - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-13 - compiler: clang - target: aarch64-apple-darwin/clang architecture: aarch64 runner: macos-14 - compiler: clang - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 runner: ubuntu-22.04 - compiler: gcc - - target: x86_64-unknown-linux-gnu/clang - architecture: x86_64 - runner: ubuntu-22.04 - compiler: clang - target: aarch64-unknown-linux-gnu/gcc architecture: aarch64 runner: ubuntu-22.04 - compiler: gcc - - target: aarch64-unknown-linux-gnu/clang - architecture: aarch64 - runner: ubuntu-22.04 - compiler: clang - env: - CC: ${{ matrix.compiler }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -111,10 +92,10 @@ jobs: if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # No PGO or tests (yet): + # No tests (yet): - name: Emulated Windows if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | @@ -132,7 +113,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -141,7 +122,7 @@ jobs: run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -156,13 +137,11 @@ jobs: make clean --jobs 4 export HOST=${{ matrix.architecture }}-linux-gnu sudo apt install --yes "gcc-$HOST" qemu-user - ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }} - ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }} export QEMU_LD_PREFIX="/usr/$HOST" - CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ - CPP="$CC --preprocess" \ + CC="$HOST-gcc" \ + CPP="$HOST-gcc --preprocess" \ HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From 3a77980002845c22e5b294ca47a12d62bf5baf53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Wed, 27 Nov 2024 23:32:54 +0000 Subject: [PATCH 039/427] GH-127178: install a _sysconfig_vars_(...).json file in the stdlib directory (#127302) --- Lib/sysconfig/__main__.py | 37 ++++++++++++++----- Lib/test/test_sysconfig.py | 33 ++++++++++++++++- Makefile.pre.in | 4 +- ...-11-26-17-42-00.gh-issue-127178.U8hxjc.rst | 4 ++ 4 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index d7257b9d2d00db..5660a6c5105b9f 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -1,5 +1,7 @@ +import json import os import sys +import types from sysconfig import ( _ALWAYS_STR, _PYTHON_BUILD, @@ -157,6 +159,19 @@ def _print_config_dict(d, stream): print ("}", file=stream) +def _get_pybuilddir(): + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + return pybuilddir + + +def _get_json_data_name(): + name = _get_sysconfigdata_name() + assert name.startswith('_sysconfigdata') + return name.replace('_sysconfigdata', '_sysconfig_vars') + '.json' + + def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" vars = {} @@ -185,6 +200,8 @@ def _generate_posix_vars(): if _PYTHON_BUILD: vars['BLDSHARED'] = vars['LDSHARED'] + name = _get_sysconfigdata_name() + # There's a chicken-and-egg situation on OS X with regards to the # _sysconfigdata module after the changes introduced by #15298: # get_config_vars() is called by get_platform() as part of the @@ -196,16 +213,13 @@ def _generate_posix_vars(): # _sysconfigdata module manually and populate it with the build vars. # This is more than sufficient for ensuring the subsequent call to # get_platform() succeeds. - name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module + # GH-127178: Since we started generating a .json file, we also need this to + # be able to run sysconfig.get_config_vars(). + module = types.ModuleType(name) + module.build_time_vars = vars + sys.modules[name] = module - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): - pybuilddir += '-pydebug' + pybuilddir = _get_pybuilddir() os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') @@ -215,6 +229,11 @@ def _generate_posix_vars(): f.write('build_time_vars = ') _print_config_dict(vars, stream=f) + # Write a JSON file with the output of sysconfig.get_config_vars + jsonfile = os.path.join(pybuilddir, _get_json_data_name()) + with open(jsonfile, 'w') as f: + json.dump(get_config_vars(), f, indent=2) + # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index a705dd0cd89e4d..0df1a67ea2b720 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -11,6 +11,7 @@ from test.support import ( captured_stdout, + is_android, is_apple_mobile, is_wasi, PythonSymlink, @@ -25,8 +26,9 @@ from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes) -from sysconfig.__main__ import _main, _parse_makefile + _expand_vars, _get_preferred_schemes, + is_python_build, _PROJECT_BASE) +from sysconfig.__main__ import _main, _parse_makefile, _get_pybuilddir, _get_json_data_name import _imp import _osx_support import _sysconfig @@ -39,6 +41,7 @@ class TestSysConfig(unittest.TestCase): def setUp(self): super(TestSysConfig, self).setUp() + self.maxDiff = None self.sys_path = sys.path[:] # patching os.uname if hasattr(os, 'uname'): @@ -625,6 +628,32 @@ def test_makefile_overwrites_config_vars(self): self.assertNotEqual(data['prefix'], data['base_prefix']) self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) + @unittest.skipIf(os.name != 'posix', '_sysconfig-vars JSON file is only available on POSIX') + @unittest.skipIf(is_wasi, "_sysconfig-vars JSON file currently isn't available on WASI") + @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS change the prefix') + def test_sysconfigdata_json(self): + if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: + data_dir = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] + elif is_python_build(): + data_dir = os.path.join(_PROJECT_BASE, _get_pybuilddir()) + else: + data_dir = sys._stdlib_dir + + json_data_path = os.path.join(data_dir, _get_json_data_name()) + + with open(json_data_path) as f: + json_config_vars = json.load(f) + + system_config_vars = get_config_vars() + + # Ignore keys in the check + for key in ('projectbase', 'srcdir'): + json_config_vars.pop(key) + system_config_vars.pop(key) + + self.assertEqual(system_config_vars, json_config_vars) + + class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), diff --git a/Makefile.pre.in b/Makefile.pre.in index 8d94ba361fd934..724354746b8d81 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2645,8 +2645,8 @@ libinstall: all $(srcdir)/Modules/xxmodule.c esac; \ done; \ done - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ - $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @ # If app store compliance has been configured, apply the patch to the @ # installed library code. The patch has been previously validated against diff --git a/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst b/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst new file mode 100644 index 00000000000000..b703b58ea8e1d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst @@ -0,0 +1,4 @@ +A ``_sysconfig_vars_(...).json`` file is now shipped in the standard library +directory. It contains the output of :func:`sysconfig.get_config_vars` on +the default environment encoded as JSON data. This is an implementation +detail, and may change at any time. From db5c5763f3e3172f1dd011355b41469770dafc0f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 28 Nov 2024 13:29:27 +0100 Subject: [PATCH 040/427] gh-127330: Update for OpenSSL 3.4 & document+improve the update process (GH-127331) - Add `git describe` output to headers generated by `make_ssl_data.py` This info is more important than the date when the file was generated. It does mean that the tool now requires a Git checkout of OpenSSL, not for example a release tarball. - Regenerate the older file to add the info. To the other older file, add a note about manual edits. - Add notes on how to add a new OpenSSL version - Add 3.4 error messages and multissl tests --- .github/workflows/build.yml | 3 +- ...-11-27-14-23-02.gh-issue-127331.9sNEC9.rst | 1 + Modules/_ssl.c | 3 +- Modules/_ssl_data_111.h | 4 +- Modules/_ssl_data_300.h | 5 +- Modules/{_ssl_data_31.h => _ssl_data_34.h} | 674 +++++++++++++++++- Tools/c-analyzer/cpython/_parser.py | 4 +- Tools/ssl/make_ssl_data.py | 34 +- Tools/ssl/multissltests.py | 2 + 9 files changed, 719 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-27-14-23-02.gh-issue-127331.9sNEC9.rst rename Modules/{_ssl_data_31.h => _ssl_data_34.h} (92%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b3c4f179b513a4..1f8c468475470c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -250,7 +250,8 @@ jobs: fail-fast: false matrix: os: [ubuntu-24.04] - openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2] + openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2, 3.4.0] + # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl diff --git a/Misc/NEWS.d/next/Library/2024-11-27-14-23-02.gh-issue-127331.9sNEC9.rst b/Misc/NEWS.d/next/Library/2024-11-27-14-23-02.gh-issue-127331.9sNEC9.rst new file mode 100644 index 00000000000000..c668816955ca59 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-27-14-23-02.gh-issue-127331.9sNEC9.rst @@ -0,0 +1 @@ +:mod:`ssl` can show descriptions for errors added in OpenSSL 3.4. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b6b5ebf094c938..e5b8bf21002ea5 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -120,8 +120,9 @@ static void _PySSLFixErrno(void) { #endif /* Include generated data (error codes) */ +/* See make_ssl_data.h for notes on adding a new version. */ #if (OPENSSL_VERSION_NUMBER >= 0x30100000L) -#include "_ssl_data_31.h" +#include "_ssl_data_34.h" #elif (OPENSSL_VERSION_NUMBER >= 0x30000000L) #include "_ssl_data_300.h" #elif (OPENSSL_VERSION_NUMBER >= 0x10101000L) diff --git a/Modules/_ssl_data_111.h b/Modules/_ssl_data_111.h index 093c786e6a26f6..061fac2bd5822b 100644 --- a/Modules/_ssl_data_111.h +++ b/Modules/_ssl_data_111.h @@ -1,4 +1,6 @@ -/* File generated by Tools/ssl/make_ssl_data.py *//* Generated on 2023-06-01T02:58:04.081473 */ +/* File generated by Tools/ssl/make_ssl_data.py */ +/* Generated on 2024-11-27T12:48:46.194048+00:00 */ +/* Generated from Git commit OpenSSL_1_1_1w-0-ge04bd3433f */ static struct py_ssl_library_code library_codes[] = { #ifdef ERR_LIB_ASN1 {"ASN1", ERR_LIB_ASN1}, diff --git a/Modules/_ssl_data_300.h b/Modules/_ssl_data_300.h index dc66731f6b6093..b687ce43c77d66 100644 --- a/Modules/_ssl_data_300.h +++ b/Modules/_ssl_data_300.h @@ -1,4 +1,7 @@ -/* File generated by Tools/ssl/make_ssl_data.py *//* Generated on 2023-06-01T03:03:52.163218 */ +/* File generated by Tools/ssl/make_ssl_data.py */ +/* Generated on 2023-06-01T03:03:52.163218 */ +/* Manually edited to add definitions from 1.1.1 (GH-105174) */ + static struct py_ssl_library_code library_codes[] = { #ifdef ERR_LIB_ASN1 {"ASN1", ERR_LIB_ASN1}, diff --git a/Modules/_ssl_data_31.h b/Modules/_ssl_data_34.h similarity index 92% rename from Modules/_ssl_data_31.h rename to Modules/_ssl_data_34.h index c589c501f4e948..d4af3e1c1fa928 100644 --- a/Modules/_ssl_data_31.h +++ b/Modules/_ssl_data_34.h @@ -1,4 +1,6 @@ -/* File generated by Tools/ssl/make_ssl_data.py *//* Generated on 2023-06-01T03:04:00.275280 */ +/* File generated by Tools/ssl/make_ssl_data.py */ +/* Generated on 2024-11-27T12:35:52.276767+00:00 */ +/* Generated from Git commit openssl-3.4.0-0-g98acb6b028 */ static struct py_ssl_library_code library_codes[] = { #ifdef ERR_LIB_ASN1 {"ASN1", ERR_LIB_ASN1}, @@ -300,6 +302,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"FIRST_NUM_TOO_LARGE", 13, 122}, #endif + #ifdef ASN1_R_GENERALIZEDTIME_IS_TOO_SHORT + {"GENERALIZEDTIME_IS_TOO_SHORT", ERR_LIB_ASN1, ASN1_R_GENERALIZEDTIME_IS_TOO_SHORT}, + #else + {"GENERALIZEDTIME_IS_TOO_SHORT", 13, 232}, + #endif #ifdef ASN1_R_HEADER_TOO_LONG {"HEADER_TOO_LONG", ERR_LIB_ASN1, ASN1_R_HEADER_TOO_LONG}, #else @@ -730,6 +737,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_TYPE", 13, 196}, #endif + #ifdef ASN1_R_UTCTIME_IS_TOO_SHORT + {"UTCTIME_IS_TOO_SHORT", ERR_LIB_ASN1, ASN1_R_UTCTIME_IS_TOO_SHORT}, + #else + {"UTCTIME_IS_TOO_SHORT", 13, 233}, + #endif #ifdef ASN1_R_WRONG_INTEGER_TYPE {"WRONG_INTEGER_TYPE", ERR_LIB_ASN1, ASN1_R_WRONG_INTEGER_TYPE}, #else @@ -845,6 +857,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"LISTEN_V6_ONLY", 32, 136}, #endif + #ifdef BIO_R_LOCAL_ADDR_NOT_AVAILABLE + {"LOCAL_ADDR_NOT_AVAILABLE", ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE}, + #else + {"LOCAL_ADDR_NOT_AVAILABLE", 32, 111}, + #endif #ifdef BIO_R_LOOKUP_RETURNED_NOTHING {"LOOKUP_RETURNED_NOTHING", ERR_LIB_BIO, BIO_R_LOOKUP_RETURNED_NOTHING}, #else @@ -860,6 +877,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NBIO_CONNECT_ERROR", 32, 110}, #endif + #ifdef BIO_R_NON_FATAL + {"NON_FATAL", ERR_LIB_BIO, BIO_R_NON_FATAL}, + #else + {"NON_FATAL", 32, 112}, + #endif #ifdef BIO_R_NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED {"NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED", ERR_LIB_BIO, BIO_R_NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED}, #else @@ -880,6 +902,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_SUCH_FILE", 32, 128}, #endif + #ifdef BIO_R_PEER_ADDR_NOT_AVAILABLE + {"PEER_ADDR_NOT_AVAILABLE", ERR_LIB_BIO, BIO_R_PEER_ADDR_NOT_AVAILABLE}, + #else + {"PEER_ADDR_NOT_AVAILABLE", 32, 114}, + #endif + #ifdef BIO_R_PORT_MISMATCH + {"PORT_MISMATCH", ERR_LIB_BIO, BIO_R_PORT_MISMATCH}, + #else + {"PORT_MISMATCH", 32, 150}, + #endif + #ifdef BIO_R_TFO_DISABLED + {"TFO_DISABLED", ERR_LIB_BIO, BIO_R_TFO_DISABLED}, + #else + {"TFO_DISABLED", 32, 106}, + #endif + #ifdef BIO_R_TFO_NO_KERNEL_SUPPORT + {"TFO_NO_KERNEL_SUPPORT", ERR_LIB_BIO, BIO_R_TFO_NO_KERNEL_SUPPORT}, + #else + {"TFO_NO_KERNEL_SUPPORT", 32, 108}, + #endif #ifdef BIO_R_TRANSFER_ERROR {"TRANSFER_ERROR", ERR_LIB_BIO, BIO_R_TRANSFER_ERROR}, #else @@ -920,6 +962,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNABLE_TO_REUSEADDR", 32, 139}, #endif + #ifdef BIO_R_UNABLE_TO_TFO + {"UNABLE_TO_TFO", ERR_LIB_BIO, BIO_R_UNABLE_TO_TFO}, + #else + {"UNABLE_TO_TFO", 32, 109}, + #endif #ifdef BIO_R_UNAVAILABLE_IP_FAMILY {"UNAVAILABLE_IP_FAMILY", ERR_LIB_BIO, BIO_R_UNAVAILABLE_IP_FAMILY}, #else @@ -1230,6 +1277,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"ERROR_VALIDATING_SIGNATURE", 58, 171}, #endif + #ifdef CMP_R_EXPECTED_POLLREQ + {"EXPECTED_POLLREQ", ERR_LIB_CMP, CMP_R_EXPECTED_POLLREQ}, + #else + {"EXPECTED_POLLREQ", 58, 104}, + #endif #ifdef CMP_R_FAILED_BUILDING_OWN_CHAIN {"FAILED_BUILDING_OWN_CHAIN", ERR_LIB_CMP, CMP_R_FAILED_BUILDING_OWN_CHAIN}, #else @@ -1250,16 +1302,51 @@ static struct py_ssl_error_code error_codes[] = { #else {"FAIL_INFO_OUT_OF_RANGE", 58, 129}, #endif + #ifdef CMP_R_GENERATE_CERTREQTEMPLATE + {"GENERATE_CERTREQTEMPLATE", ERR_LIB_CMP, CMP_R_GENERATE_CERTREQTEMPLATE}, + #else + {"GENERATE_CERTREQTEMPLATE", 58, 197}, + #endif + #ifdef CMP_R_GENERATE_CRLSTATUS + {"GENERATE_CRLSTATUS", ERR_LIB_CMP, CMP_R_GENERATE_CRLSTATUS}, + #else + {"GENERATE_CRLSTATUS", 58, 198}, + #endif + #ifdef CMP_R_GETTING_GENP + {"GETTING_GENP", ERR_LIB_CMP, CMP_R_GETTING_GENP}, + #else + {"GETTING_GENP", 58, 192}, + #endif + #ifdef CMP_R_GET_ITAV + {"GET_ITAV", ERR_LIB_CMP, CMP_R_GET_ITAV}, + #else + {"GET_ITAV", 58, 199}, + #endif #ifdef CMP_R_INVALID_ARGS {"INVALID_ARGS", ERR_LIB_CMP, CMP_R_INVALID_ARGS}, #else {"INVALID_ARGS", 58, 100}, #endif + #ifdef CMP_R_INVALID_GENP + {"INVALID_GENP", ERR_LIB_CMP, CMP_R_INVALID_GENP}, + #else + {"INVALID_GENP", 58, 193}, + #endif + #ifdef CMP_R_INVALID_KEYSPEC + {"INVALID_KEYSPEC", ERR_LIB_CMP, CMP_R_INVALID_KEYSPEC}, + #else + {"INVALID_KEYSPEC", 58, 202}, + #endif #ifdef CMP_R_INVALID_OPTION {"INVALID_OPTION", ERR_LIB_CMP, CMP_R_INVALID_OPTION}, #else {"INVALID_OPTION", 58, 174}, #endif + #ifdef CMP_R_INVALID_ROOTCAKEYUPDATE + {"INVALID_ROOTCAKEYUPDATE", ERR_LIB_CMP, CMP_R_INVALID_ROOTCAKEYUPDATE}, + #else + {"INVALID_ROOTCAKEYUPDATE", 58, 195}, + #endif #ifdef CMP_R_MISSING_CERTID {"MISSING_CERTID", ERR_LIB_CMP, CMP_R_MISSING_CERTID}, #else @@ -1425,6 +1512,21 @@ static struct py_ssl_error_code error_codes[] = { #else {"TRANSFER_ERROR", 58, 159}, #endif + #ifdef CMP_R_UNCLEAN_CTX + {"UNCLEAN_CTX", ERR_LIB_CMP, CMP_R_UNCLEAN_CTX}, + #else + {"UNCLEAN_CTX", 58, 191}, + #endif + #ifdef CMP_R_UNEXPECTED_CERTPROFILE + {"UNEXPECTED_CERTPROFILE", ERR_LIB_CMP, CMP_R_UNEXPECTED_CERTPROFILE}, + #else + {"UNEXPECTED_CERTPROFILE", 58, 196}, + #endif + #ifdef CMP_R_UNEXPECTED_CRLSTATUSLIST + {"UNEXPECTED_CRLSTATUSLIST", ERR_LIB_CMP, CMP_R_UNEXPECTED_CRLSTATUSLIST}, + #else + {"UNEXPECTED_CRLSTATUSLIST", 58, 201}, + #endif #ifdef CMP_R_UNEXPECTED_PKIBODY {"UNEXPECTED_PKIBODY", ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY}, #else @@ -1435,11 +1537,21 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNEXPECTED_PKISTATUS", 58, 185}, #endif + #ifdef CMP_R_UNEXPECTED_POLLREQ + {"UNEXPECTED_POLLREQ", ERR_LIB_CMP, CMP_R_UNEXPECTED_POLLREQ}, + #else + {"UNEXPECTED_POLLREQ", 58, 105}, + #endif #ifdef CMP_R_UNEXPECTED_PVNO {"UNEXPECTED_PVNO", ERR_LIB_CMP, CMP_R_UNEXPECTED_PVNO}, #else {"UNEXPECTED_PVNO", 58, 153}, #endif + #ifdef CMP_R_UNEXPECTED_SENDER + {"UNEXPECTED_SENDER", ERR_LIB_CMP, CMP_R_UNEXPECTED_SENDER}, + #else + {"UNEXPECTED_SENDER", 58, 106}, + #endif #ifdef CMP_R_UNKNOWN_ALGORITHM_ID {"UNKNOWN_ALGORITHM_ID", ERR_LIB_CMP, CMP_R_UNKNOWN_ALGORITHM_ID}, #else @@ -1450,6 +1562,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNKNOWN_CERT_TYPE", 58, 135}, #endif + #ifdef CMP_R_UNKNOWN_CRL_ISSUER + {"UNKNOWN_CRL_ISSUER", ERR_LIB_CMP, CMP_R_UNKNOWN_CRL_ISSUER}, + #else + {"UNKNOWN_CRL_ISSUER", 58, 200}, + #endif #ifdef CMP_R_UNKNOWN_PKISTATUS {"UNKNOWN_PKISTATUS", ERR_LIB_CMP, CMP_R_UNKNOWN_PKISTATUS}, #else @@ -1465,6 +1582,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_KEY_TYPE", 58, 137}, #endif + #ifdef CMP_R_UNSUPPORTED_PKIBODY + {"UNSUPPORTED_PKIBODY", ERR_LIB_CMP, CMP_R_UNSUPPORTED_PKIBODY}, + #else + {"UNSUPPORTED_PKIBODY", 58, 101}, + #endif #ifdef CMP_R_UNSUPPORTED_PROTECTION_ALG_DHBASEDMAC {"UNSUPPORTED_PROTECTION_ALG_DHBASEDMAC", ERR_LIB_CMP, CMP_R_UNSUPPORTED_PROTECTION_ALG_DHBASEDMAC}, #else @@ -1825,6 +1947,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_SIGNERS", 46, 135}, #endif + #ifdef CMS_R_OPERATION_UNSUPPORTED + {"OPERATION_UNSUPPORTED", ERR_LIB_CMS, CMS_R_OPERATION_UNSUPPORTED}, + #else + {"OPERATION_UNSUPPORTED", 46, 182}, + #endif #ifdef CMS_R_PEER_KEY_ERROR {"PEER_KEY_ERROR", ERR_LIB_CMS, CMS_R_PEER_KEY_ERROR}, #else @@ -1960,6 +2087,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_RECIPIENT_TYPE", 46, 154}, #endif + #ifdef CMS_R_UNSUPPORTED_SIGNATURE_ALGORITHM + {"UNSUPPORTED_SIGNATURE_ALGORITHM", ERR_LIB_CMS, CMS_R_UNSUPPORTED_SIGNATURE_ALGORITHM}, + #else + {"UNSUPPORTED_SIGNATURE_ALGORITHM", 46, 195}, + #endif #ifdef CMS_R_UNSUPPORTED_TYPE {"UNSUPPORTED_TYPE", ERR_LIB_CMS, CMS_R_UNSUPPORTED_TYPE}, #else @@ -1985,6 +2117,31 @@ static struct py_ssl_error_code error_codes[] = { #else {"WRAP_ERROR", 46, 159}, #endif + #ifdef COMP_R_BROTLI_DECODE_ERROR + {"BROTLI_DECODE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_DECODE_ERROR}, + #else + {"BROTLI_DECODE_ERROR", 41, 102}, + #endif + #ifdef COMP_R_BROTLI_DEFLATE_ERROR + {"BROTLI_DEFLATE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_DEFLATE_ERROR}, + #else + {"BROTLI_DEFLATE_ERROR", 41, 103}, + #endif + #ifdef COMP_R_BROTLI_ENCODE_ERROR + {"BROTLI_ENCODE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_ENCODE_ERROR}, + #else + {"BROTLI_ENCODE_ERROR", 41, 106}, + #endif + #ifdef COMP_R_BROTLI_INFLATE_ERROR + {"BROTLI_INFLATE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_INFLATE_ERROR}, + #else + {"BROTLI_INFLATE_ERROR", 41, 104}, + #endif + #ifdef COMP_R_BROTLI_NOT_SUPPORTED + {"BROTLI_NOT_SUPPORTED", ERR_LIB_COMP, COMP_R_BROTLI_NOT_SUPPORTED}, + #else + {"BROTLI_NOT_SUPPORTED", 41, 105}, + #endif #ifdef COMP_R_ZLIB_DEFLATE_ERROR {"ZLIB_DEFLATE_ERROR", ERR_LIB_COMP, COMP_R_ZLIB_DEFLATE_ERROR}, #else @@ -2000,6 +2157,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"ZLIB_NOT_SUPPORTED", 41, 101}, #endif + #ifdef COMP_R_ZSTD_COMPRESS_ERROR + {"ZSTD_COMPRESS_ERROR", ERR_LIB_COMP, COMP_R_ZSTD_COMPRESS_ERROR}, + #else + {"ZSTD_COMPRESS_ERROR", 41, 107}, + #endif + #ifdef COMP_R_ZSTD_DECODE_ERROR + {"ZSTD_DECODE_ERROR", ERR_LIB_COMP, COMP_R_ZSTD_DECODE_ERROR}, + #else + {"ZSTD_DECODE_ERROR", 41, 108}, + #endif + #ifdef COMP_R_ZSTD_DECOMPRESS_ERROR + {"ZSTD_DECOMPRESS_ERROR", ERR_LIB_COMP, COMP_R_ZSTD_DECOMPRESS_ERROR}, + #else + {"ZSTD_DECOMPRESS_ERROR", 41, 109}, + #endif + #ifdef COMP_R_ZSTD_NOT_SUPPORTED + {"ZSTD_NOT_SUPPORTED", ERR_LIB_COMP, COMP_R_ZSTD_NOT_SUPPORTED}, + #else + {"ZSTD_NOT_SUPPORTED", 41, 110}, + #endif #ifdef CONF_R_ERROR_LOADING_DSO {"ERROR_LOADING_DSO", ERR_LIB_CONF, CONF_R_ERROR_LOADING_DSO}, #else @@ -2085,6 +2262,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"RECURSIVE_DIRECTORY_INCLUDE", 14, 111}, #endif + #ifdef CONF_R_RECURSIVE_SECTION_REFERENCE + {"RECURSIVE_SECTION_REFERENCE", ERR_LIB_CONF, CONF_R_RECURSIVE_SECTION_REFERENCE}, + #else + {"RECURSIVE_SECTION_REFERENCE", 14, 126}, + #endif #ifdef CONF_R_RELATIVE_PATH {"RELATIVE_PATH", ERR_LIB_CONF, CONF_R_RELATIVE_PATH}, #else @@ -2370,6 +2552,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"TOO_MANY_BYTES", 15, 113}, #endif + #ifdef CRYPTO_R_TOO_MANY_NAMES + {"TOO_MANY_NAMES", ERR_LIB_CRYPTO, CRYPTO_R_TOO_MANY_NAMES}, + #else + {"TOO_MANY_NAMES", 15, 132}, + #endif #ifdef CRYPTO_R_TOO_MANY_RECORDS {"TOO_MANY_RECORDS", ERR_LIB_CRYPTO, CRYPTO_R_TOO_MANY_RECORDS}, #else @@ -2560,6 +2747,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_SECRET", 5, 128}, #endif + #ifdef DH_R_INVALID_SIZE + {"INVALID_SIZE", ERR_LIB_DH, DH_R_INVALID_SIZE}, + #else + {"INVALID_SIZE", 5, 129}, + #endif #ifdef DH_R_KDF_PARAMETER_ERROR {"KDF_PARAMETER_ERROR", ERR_LIB_DH, DH_R_KDF_PARAMETER_ERROR}, #else @@ -2610,6 +2802,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"PEER_KEY_ERROR", 5, 111}, #endif + #ifdef DH_R_Q_TOO_LARGE + {"Q_TOO_LARGE", ERR_LIB_DH, DH_R_Q_TOO_LARGE}, + #else + {"Q_TOO_LARGE", 5, 130}, + #endif #ifdef DH_R_SHARED_INFO_ERROR {"SHARED_INFO_ERROR", ERR_LIB_DH, DH_R_SHARED_INFO_ERROR}, #else @@ -3545,6 +3742,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"GENERATE_ERROR", 6, 214}, #endif + #ifdef EVP_R_GETTING_ALGORITHMIDENTIFIER_NOT_SUPPORTED + {"GETTING_ALGORITHMIDENTIFIER_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_GETTING_ALGORITHMIDENTIFIER_NOT_SUPPORTED}, + #else + {"GETTING_ALGORITHMIDENTIFIER_NOT_SUPPORTED", 6, 229}, + #endif #ifdef EVP_R_GET_RAW_KEY_FAILED {"GET_RAW_KEY_FAILED", ERR_LIB_EVP, EVP_R_GET_RAW_KEY_FAILED}, #else @@ -3745,6 +3947,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", 6, 150}, #endif + #ifdef EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_SIGNATURE_TYPE + {"OPERATION_NOT_SUPPORTED_FOR_THIS_SIGNATURE_TYPE", ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_SIGNATURE_TYPE}, + #else + {"OPERATION_NOT_SUPPORTED_FOR_THIS_SIGNATURE_TYPE", 6, 226}, + #endif #ifdef EVP_R_OUTPUT_WOULD_OVERFLOW {"OUTPUT_WOULD_OVERFLOW", ERR_LIB_EVP, EVP_R_OUTPUT_WOULD_OVERFLOW}, #else @@ -3795,6 +4002,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"SET_DEFAULT_PROPERTY_FAILURE", 6, 209}, #endif + #ifdef EVP_R_SIGNATURE_TYPE_AND_KEY_TYPE_INCOMPATIBLE + {"SIGNATURE_TYPE_AND_KEY_TYPE_INCOMPATIBLE", ERR_LIB_EVP, EVP_R_SIGNATURE_TYPE_AND_KEY_TYPE_INCOMPATIBLE}, + #else + {"SIGNATURE_TYPE_AND_KEY_TYPE_INCOMPATIBLE", 6, 228}, + #endif #ifdef EVP_R_TOO_MANY_RECORDS {"TOO_MANY_RECORDS", ERR_LIB_EVP, EVP_R_TOO_MANY_RECORDS}, #else @@ -3825,6 +4037,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNABLE_TO_SET_CALLBACKS", 6, 217}, #endif + #ifdef EVP_R_UNKNOWN_BITS + {"UNKNOWN_BITS", ERR_LIB_EVP, EVP_R_UNKNOWN_BITS}, + #else + {"UNKNOWN_BITS", 6, 166}, + #endif #ifdef EVP_R_UNKNOWN_CIPHER {"UNKNOWN_CIPHER", ERR_LIB_EVP, EVP_R_UNKNOWN_CIPHER}, #else @@ -3840,6 +4057,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNKNOWN_KEY_TYPE", 6, 207}, #endif + #ifdef EVP_R_UNKNOWN_MAX_SIZE + {"UNKNOWN_MAX_SIZE", ERR_LIB_EVP, EVP_R_UNKNOWN_MAX_SIZE}, + #else + {"UNKNOWN_MAX_SIZE", 6, 167}, + #endif #ifdef EVP_R_UNKNOWN_OPTION {"UNKNOWN_OPTION", ERR_LIB_EVP, EVP_R_UNKNOWN_OPTION}, #else @@ -3850,6 +4072,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNKNOWN_PBE_ALGORITHM", 6, 121}, #endif + #ifdef EVP_R_UNKNOWN_SECURITY_BITS + {"UNKNOWN_SECURITY_BITS", ERR_LIB_EVP, EVP_R_UNKNOWN_SECURITY_BITS}, + #else + {"UNKNOWN_SECURITY_BITS", 6, 168}, + #endif #ifdef EVP_R_UNSUPPORTED_ALGORITHM {"UNSUPPORTED_ALGORITHM", ERR_LIB_EVP, EVP_R_UNSUPPORTED_ALGORITHM}, #else @@ -4040,6 +4267,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"RESPONSE_PARSE_ERROR", 61, 104}, #endif + #ifdef HTTP_R_RESPONSE_TOO_MANY_HDRLINES + {"RESPONSE_TOO_MANY_HDRLINES", ERR_LIB_HTTP, HTTP_R_RESPONSE_TOO_MANY_HDRLINES}, + #else + {"RESPONSE_TOO_MANY_HDRLINES", 61, 130}, + #endif #ifdef HTTP_R_RETRY_TIMEOUT {"RETRY_TIMEOUT", ERR_LIB_HTTP, HTTP_R_RETRY_TIMEOUT}, #else @@ -4530,6 +4762,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_PUBLIC_KEY_TYPE", 9, 110}, #endif + #ifdef PKCS12_R_CALLBACK_FAILED + {"CALLBACK_FAILED", ERR_LIB_PKCS12, PKCS12_R_CALLBACK_FAILED}, + #else + {"CALLBACK_FAILED", 35, 115}, + #endif #ifdef PKCS12_R_CANT_PACK_STRUCTURE {"CANT_PACK_STRUCTURE", ERR_LIB_PKCS12, PKCS12_R_CANT_PACK_STRUCTURE}, #else @@ -4920,6 +5157,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"CIPHER_OPERATION_FAILED", 57, 102}, #endif + #ifdef PROV_R_COFACTOR_REQUIRED + {"COFACTOR_REQUIRED", ERR_LIB_PROV, PROV_R_COFACTOR_REQUIRED}, + #else + {"COFACTOR_REQUIRED", 57, 236}, + #endif #ifdef PROV_R_DERIVATION_FUNCTION_INIT_FAILED {"DERIVATION_FUNCTION_INIT_FAILED", ERR_LIB_PROV, PROV_R_DERIVATION_FUNCTION_INIT_FAILED}, #else @@ -4935,6 +5177,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"EMS_NOT_ENABLED", 57, 233}, #endif + #ifdef PROV_R_ENTROPY_SOURCE_FAILED_CONTINUOUS_TESTS + {"ENTROPY_SOURCE_FAILED_CONTINUOUS_TESTS", ERR_LIB_PROV, PROV_R_ENTROPY_SOURCE_FAILED_CONTINUOUS_TESTS}, + #else + {"ENTROPY_SOURCE_FAILED_CONTINUOUS_TESTS", 57, 244}, + #endif #ifdef PROV_R_ENTROPY_SOURCE_STRENGTH_TOO_WEAK {"ENTROPY_SOURCE_STRENGTH_TOO_WEAK", ERR_LIB_PROV, PROV_R_ENTROPY_SOURCE_STRENGTH_TOO_WEAK}, #else @@ -4990,6 +5237,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"FAILED_TO_SIGN", 57, 175}, #endif + #ifdef PROV_R_FINAL_CALL_OUT_OF_ORDER + {"FINAL_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_FINAL_CALL_OUT_OF_ORDER}, + #else + {"FINAL_CALL_OUT_OF_ORDER", 57, 237}, + #endif #ifdef PROV_R_FIPS_MODULE_CONDITIONAL_ERROR {"FIPS_MODULE_CONDITIONAL_ERROR", ERR_LIB_PROV, PROV_R_FIPS_MODULE_CONDITIONAL_ERROR}, #else @@ -5020,6 +5272,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INDICATOR_INTEGRITY_FAILURE", 57, 210}, #endif + #ifdef PROV_R_INIT_CALL_OUT_OF_ORDER + {"INIT_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_INIT_CALL_OUT_OF_ORDER}, + #else + {"INIT_CALL_OUT_OF_ORDER", 57, 238}, + #endif #ifdef PROV_R_INSUFFICIENT_DRBG_STRENGTH {"INSUFFICIENT_DRBG_STRENGTH", ERR_LIB_PROV, PROV_R_INSUFFICIENT_DRBG_STRENGTH}, #else @@ -5030,6 +5287,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_AAD", 57, 108}, #endif + #ifdef PROV_R_INVALID_AEAD + {"INVALID_AEAD", ERR_LIB_PROV, PROV_R_INVALID_AEAD}, + #else + {"INVALID_AEAD", 57, 231}, + #endif #ifdef PROV_R_INVALID_CONFIG_DATA {"INVALID_CONFIG_DATA", ERR_LIB_PROV, PROV_R_INVALID_CONFIG_DATA}, #else @@ -5070,6 +5332,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_DIGEST_SIZE", 57, 218}, #endif + #ifdef PROV_R_INVALID_EDDSA_INSTANCE_FOR_ATTEMPTED_OPERATION + {"INVALID_EDDSA_INSTANCE_FOR_ATTEMPTED_OPERATION", ERR_LIB_PROV, PROV_R_INVALID_EDDSA_INSTANCE_FOR_ATTEMPTED_OPERATION}, + #else + {"INVALID_EDDSA_INSTANCE_FOR_ATTEMPTED_OPERATION", 57, 243}, + #endif #ifdef PROV_R_INVALID_INPUT_LENGTH {"INVALID_INPUT_LENGTH", ERR_LIB_PROV, PROV_R_INVALID_INPUT_LENGTH}, #else @@ -5085,6 +5352,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_IV_LENGTH", 57, 109}, #endif + #ifdef PROV_R_INVALID_KDF + {"INVALID_KDF", ERR_LIB_PROV, PROV_R_INVALID_KDF}, + #else + {"INVALID_KDF", 57, 232}, + #endif #ifdef PROV_R_INVALID_KEY {"INVALID_KEY", ERR_LIB_PROV, PROV_R_INVALID_KEY}, #else @@ -5100,6 +5372,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_MAC", 57, 151}, #endif + #ifdef PROV_R_INVALID_MEMORY_SIZE + {"INVALID_MEMORY_SIZE", ERR_LIB_PROV, PROV_R_INVALID_MEMORY_SIZE}, + #else + {"INVALID_MEMORY_SIZE", 57, 235}, + #endif #ifdef PROV_R_INVALID_MGF1_MD {"INVALID_MGF1_MD", ERR_LIB_PROV, PROV_R_INVALID_MGF1_MD}, #else @@ -5120,6 +5397,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_PADDING_MODE", 57, 168}, #endif + #ifdef PROV_R_INVALID_PREHASHED_DIGEST_LENGTH + {"INVALID_PREHASHED_DIGEST_LENGTH", ERR_LIB_PROV, PROV_R_INVALID_PREHASHED_DIGEST_LENGTH}, + #else + {"INVALID_PREHASHED_DIGEST_LENGTH", 57, 241}, + #endif #ifdef PROV_R_INVALID_PUBINFO {"INVALID_PUBINFO", ERR_LIB_PROV, PROV_R_INVALID_PUBINFO}, #else @@ -5155,6 +5437,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_TAG_LENGTH", 57, 118}, #endif + #ifdef PROV_R_INVALID_THREAD_POOL_SIZE + {"INVALID_THREAD_POOL_SIZE", ERR_LIB_PROV, PROV_R_INVALID_THREAD_POOL_SIZE}, + #else + {"INVALID_THREAD_POOL_SIZE", 57, 234}, + #endif #ifdef PROV_R_INVALID_UKM_LENGTH {"INVALID_UKM_LENGTH", ERR_LIB_PROV, PROV_R_INVALID_UKM_LENGTH}, #else @@ -5300,6 +5587,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NOT_XOF_OR_INVALID_LENGTH", 57, 113}, #endif + #ifdef PROV_R_NO_INSTANCE_ALLOWED + {"NO_INSTANCE_ALLOWED", ERR_LIB_PROV, PROV_R_NO_INSTANCE_ALLOWED}, + #else + {"NO_INSTANCE_ALLOWED", 57, 242}, + #endif #ifdef PROV_R_NO_KEY_SET {"NO_KEY_SET", ERR_LIB_PROV, PROV_R_NO_KEY_SET}, #else @@ -5310,6 +5602,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_PARAMETERS_SET", 57, 177}, #endif + #ifdef PROV_R_ONESHOT_CALL_OUT_OF_ORDER + {"ONESHOT_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_ONESHOT_CALL_OUT_OF_ORDER}, + #else + {"ONESHOT_CALL_OUT_OF_ORDER", 57, 239}, + #endif #ifdef PROV_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", ERR_LIB_PROV, PROV_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE}, #else @@ -5460,6 +5757,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_NUMBER_OF_ROUNDS", 57, 152}, #endif + #ifdef PROV_R_UPDATE_CALL_OUT_OF_ORDER + {"UPDATE_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_UPDATE_CALL_OUT_OF_ORDER}, + #else + {"UPDATE_CALL_OUT_OF_ORDER", 57, 240}, + #endif #ifdef PROV_R_URI_AUTHORITY_UNSUPPORTED {"URI_AUTHORITY_UNSUPPORTED", ERR_LIB_PROV, PROV_R_URI_AUTHORITY_UNSUPPORTED}, #else @@ -5595,6 +5897,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INTERNAL_ERROR", 36, 113}, #endif + #ifdef RAND_R_INVALID_PROPERTY_QUERY + {"INVALID_PROPERTY_QUERY", ERR_LIB_RAND, RAND_R_INVALID_PROPERTY_QUERY}, + #else + {"INVALID_PROPERTY_QUERY", 36, 137}, + #endif #ifdef RAND_R_IN_ERROR_STATE {"IN_ERROR_STATE", ERR_LIB_RAND, RAND_R_IN_ERROR_STATE}, #else @@ -6210,6 +6517,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE", 20, 158}, #endif + #ifdef SSL_R_BAD_CERTIFICATE + {"BAD_CERTIFICATE", ERR_LIB_SSL, SSL_R_BAD_CERTIFICATE}, + #else + {"BAD_CERTIFICATE", 20, 348}, + #endif #ifdef SSL_R_BAD_CHANGE_CIPHER_SPEC {"BAD_CHANGE_CIPHER_SPEC", ERR_LIB_SSL, SSL_R_BAD_CHANGE_CIPHER_SPEC}, #else @@ -6220,6 +6532,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"BAD_CIPHER", 20, 186}, #endif + #ifdef SSL_R_BAD_COMPRESSION_ALGORITHM + {"BAD_COMPRESSION_ALGORITHM", ERR_LIB_SSL, SSL_R_BAD_COMPRESSION_ALGORITHM}, + #else + {"BAD_COMPRESSION_ALGORITHM", 20, 326}, + #endif #ifdef SSL_R_BAD_DATA {"BAD_DATA", ERR_LIB_SSL, SSL_R_BAD_DATA}, #else @@ -6495,6 +6812,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"CONNECTION_TYPE_NOT_SET", 20, 144}, #endif + #ifdef SSL_R_CONN_USE_ONLY + {"CONN_USE_ONLY", ERR_LIB_SSL, SSL_R_CONN_USE_ONLY}, + #else + {"CONN_USE_ONLY", 20, 356}, + #endif #ifdef SSL_R_CONTEXT_NOT_DANE_ENABLED {"CONTEXT_NOT_DANE_ENABLED", ERR_LIB_SSL, SSL_R_CONTEXT_NOT_DANE_ENABLED}, #else @@ -6635,6 +6957,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"EE_KEY_TOO_SMALL", 20, 399}, #endif + #ifdef SSL_R_EMPTY_RAW_PUBLIC_KEY + {"EMPTY_RAW_PUBLIC_KEY", ERR_LIB_SSL, SSL_R_EMPTY_RAW_PUBLIC_KEY}, + #else + {"EMPTY_RAW_PUBLIC_KEY", 20, 349}, + #endif #ifdef SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST {"EMPTY_SRTP_PROTECTION_PROFILE_LIST", ERR_LIB_SSL, SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST}, #else @@ -6650,6 +6977,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"ERROR_IN_RECEIVED_CIPHER_LIST", 20, 151}, #endif + #ifdef SSL_R_ERROR_IN_SYSTEM_DEFAULT_CONFIG + {"ERROR_IN_SYSTEM_DEFAULT_CONFIG", ERR_LIB_SSL, SSL_R_ERROR_IN_SYSTEM_DEFAULT_CONFIG}, + #else + {"ERROR_IN_SYSTEM_DEFAULT_CONFIG", 20, 419}, + #endif #ifdef SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN {"ERROR_SETTING_TLSA_BASE_DOMAIN", ERR_LIB_SSL, SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN}, #else @@ -6680,11 +7012,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"EXT_LENGTH_MISMATCH", 20, 163}, #endif + #ifdef SSL_R_FAILED_TO_GET_PARAMETER + {"FAILED_TO_GET_PARAMETER", ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER}, + #else + {"FAILED_TO_GET_PARAMETER", 20, 316}, + #endif #ifdef SSL_R_FAILED_TO_INIT_ASYNC {"FAILED_TO_INIT_ASYNC", ERR_LIB_SSL, SSL_R_FAILED_TO_INIT_ASYNC}, #else {"FAILED_TO_INIT_ASYNC", 20, 405}, #endif + #ifdef SSL_R_FEATURE_NEGOTIATION_NOT_COMPLETE + {"FEATURE_NEGOTIATION_NOT_COMPLETE", ERR_LIB_SSL, SSL_R_FEATURE_NEGOTIATION_NOT_COMPLETE}, + #else + {"FEATURE_NEGOTIATION_NOT_COMPLETE", 20, 417}, + #endif + #ifdef SSL_R_FEATURE_NOT_RENEGOTIABLE + {"FEATURE_NOT_RENEGOTIABLE", ERR_LIB_SSL, SSL_R_FEATURE_NOT_RENEGOTIABLE}, + #else + {"FEATURE_NOT_RENEGOTIABLE", 20, 413}, + #endif #ifdef SSL_R_FRAGMENTED_CLIENT_HELLO {"FRAGMENTED_CLIENT_HELLO", ERR_LIB_SSL, SSL_R_FRAGMENTED_CLIENT_HELLO}, #else @@ -6805,6 +7152,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_NULL_CMD_NAME", 20, 385}, #endif + #ifdef SSL_R_INVALID_RAW_PUBLIC_KEY + {"INVALID_RAW_PUBLIC_KEY", ERR_LIB_SSL, SSL_R_INVALID_RAW_PUBLIC_KEY}, + #else + {"INVALID_RAW_PUBLIC_KEY", 20, 350}, + #endif + #ifdef SSL_R_INVALID_RECORD + {"INVALID_RECORD", ERR_LIB_SSL, SSL_R_INVALID_RECORD}, + #else + {"INVALID_RECORD", 20, 317}, + #endif #ifdef SSL_R_INVALID_SEQUENCE_NUMBER {"INVALID_SEQUENCE_NUMBER", ERR_LIB_SSL, SSL_R_INVALID_SEQUENCE_NUMBER}, #else @@ -6865,6 +7222,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"LIBRARY_HAS_NO_CIPHERS", 20, 161}, #endif + #ifdef SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED + {"MAXIMUM_ENCRYPTED_PKTS_REACHED", ERR_LIB_SSL, SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED}, + #else + {"MAXIMUM_ENCRYPTED_PKTS_REACHED", 20, 395}, + #endif #ifdef SSL_R_MISSING_DSA_SIGNING_CERT {"MISSING_DSA_SIGNING_CERT", ERR_LIB_SSL, SSL_R_MISSING_DSA_SIGNING_CERT}, #else @@ -6925,6 +7287,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"MISSING_SUPPORTED_GROUPS_EXTENSION", 20, 209}, #endif + #ifdef SSL_R_MISSING_SUPPORTED_VERSIONS_EXTENSION + {"MISSING_SUPPORTED_VERSIONS_EXTENSION", ERR_LIB_SSL, SSL_R_MISSING_SUPPORTED_VERSIONS_EXTENSION}, + #else + {"MISSING_SUPPORTED_VERSIONS_EXTENSION", 20, 420}, + #endif #ifdef SSL_R_MISSING_TMP_DH_KEY {"MISSING_TMP_DH_KEY", ERR_LIB_SSL, SSL_R_MISSING_TMP_DH_KEY}, #else @@ -7065,6 +7432,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_SRTP_PROFILES", 20, 359}, #endif + #ifdef SSL_R_NO_STREAM + {"NO_STREAM", ERR_LIB_SSL, SSL_R_NO_STREAM}, + #else + {"NO_STREAM", 20, 355}, + #endif #ifdef SSL_R_NO_SUITABLE_DIGEST_ALGORITHM {"NO_SUITABLE_DIGEST_ALGORITHM", ERR_LIB_SSL, SSL_R_NO_SUITABLE_DIGEST_ALGORITHM}, #else @@ -7080,6 +7452,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_SUITABLE_KEY_SHARE", 20, 101}, #endif + #ifdef SSL_R_NO_SUITABLE_RECORD_LAYER + {"NO_SUITABLE_RECORD_LAYER", ERR_LIB_SSL, SSL_R_NO_SUITABLE_RECORD_LAYER}, + #else + {"NO_SUITABLE_RECORD_LAYER", 20, 322}, + #endif #ifdef SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM {"NO_SUITABLE_SIGNATURE_ALGORITHM", ERR_LIB_SSL, SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM}, #else @@ -7160,6 +7537,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"PIPELINE_FAILURE", 20, 406}, #endif + #ifdef SSL_R_POLL_REQUEST_NOT_SUPPORTED + {"POLL_REQUEST_NOT_SUPPORTED", ERR_LIB_SSL, SSL_R_POLL_REQUEST_NOT_SUPPORTED}, + #else + {"POLL_REQUEST_NOT_SUPPORTED", 20, 418}, + #endif #ifdef SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR {"POST_HANDSHAKE_AUTH_ENCODING_ERR", ERR_LIB_SSL, SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR}, #else @@ -7190,6 +7572,21 @@ static struct py_ssl_error_code error_codes[] = { #else {"PSK_NO_SERVER_CB", 20, 225}, #endif + #ifdef SSL_R_QUIC_HANDSHAKE_LAYER_ERROR + {"QUIC_HANDSHAKE_LAYER_ERROR", ERR_LIB_SSL, SSL_R_QUIC_HANDSHAKE_LAYER_ERROR}, + #else + {"QUIC_HANDSHAKE_LAYER_ERROR", 20, 393}, + #endif + #ifdef SSL_R_QUIC_NETWORK_ERROR + {"QUIC_NETWORK_ERROR", ERR_LIB_SSL, SSL_R_QUIC_NETWORK_ERROR}, + #else + {"QUIC_NETWORK_ERROR", 20, 387}, + #endif + #ifdef SSL_R_QUIC_PROTOCOL_ERROR + {"QUIC_PROTOCOL_ERROR", ERR_LIB_SSL, SSL_R_QUIC_PROTOCOL_ERROR}, + #else + {"QUIC_PROTOCOL_ERROR", 20, 382}, + #endif #ifdef SSL_R_READ_BIO_NOT_SET {"READ_BIO_NOT_SET", ERR_LIB_SSL, SSL_R_READ_BIO_NOT_SET}, #else @@ -7200,6 +7597,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"READ_TIMEOUT_EXPIRED", 20, 312}, #endif + #ifdef SSL_R_RECORDS_NOT_RELEASED + {"RECORDS_NOT_RELEASED", ERR_LIB_SSL, SSL_R_RECORDS_NOT_RELEASED}, + #else + {"RECORDS_NOT_RELEASED", 20, 321}, + #endif + #ifdef SSL_R_RECORD_LAYER_FAILURE + {"RECORD_LAYER_FAILURE", ERR_LIB_SSL, SSL_R_RECORD_LAYER_FAILURE}, + #else + {"RECORD_LAYER_FAILURE", 20, 313}, + #endif #ifdef SSL_R_RECORD_LENGTH_MISMATCH {"RECORD_LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_RECORD_LENGTH_MISMATCH}, #else @@ -7210,6 +7617,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"RECORD_TOO_SMALL", 20, 298}, #endif + #ifdef SSL_R_REMOTE_PEER_ADDRESS_NOT_SET + {"REMOTE_PEER_ADDRESS_NOT_SET", ERR_LIB_SSL, SSL_R_REMOTE_PEER_ADDRESS_NOT_SET}, + #else + {"REMOTE_PEER_ADDRESS_NOT_SET", 20, 346}, + #endif #ifdef SSL_R_RENEGOTIATE_EXT_TOO_LONG {"RENEGOTIATE_EXT_TOO_LONG", ERR_LIB_SSL, SSL_R_RENEGOTIATE_EXT_TOO_LONG}, #else @@ -7255,6 +7667,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"SCT_VERIFICATION_FAILED", 20, 208}, #endif + #ifdef SSL_R_SEQUENCE_CTR_WRAPPED + {"SEQUENCE_CTR_WRAPPED", ERR_LIB_SSL, SSL_R_SEQUENCE_CTR_WRAPPED}, + #else + {"SEQUENCE_CTR_WRAPPED", 20, 327}, + #endif #ifdef SSL_R_SERVERHELLO_TLSEXT {"SERVERHELLO_TLSEXT", ERR_LIB_SSL, SSL_R_SERVERHELLO_TLSEXT}, #else @@ -7325,6 +7742,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"SSLV3_ALERT_BAD_CERTIFICATE", 20, 1042}, #endif + #ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE + {"SSLV3_ALERT_BAD_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_BAD_CERTIFICATE}, + #else + {"SSLV3_ALERT_BAD_CERTIFICATE", 20, 1042}, + #endif + #ifdef SSL_R_SSLV3_ALERT_BAD_RECORD_MAC + {"SSLV3_ALERT_BAD_RECORD_MAC", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_BAD_RECORD_MAC}, + #else + {"SSLV3_ALERT_BAD_RECORD_MAC", 20, 1020}, + #endif #ifdef SSL_R_SSLV3_ALERT_BAD_RECORD_MAC {"SSLV3_ALERT_BAD_RECORD_MAC", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_BAD_RECORD_MAC}, #else @@ -7335,11 +7762,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"SSLV3_ALERT_CERTIFICATE_EXPIRED", 20, 1045}, #endif + #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED + {"SSLV3_ALERT_CERTIFICATE_EXPIRED", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED}, + #else + {"SSLV3_ALERT_CERTIFICATE_EXPIRED", 20, 1045}, + #endif #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED {"SSLV3_ALERT_CERTIFICATE_REVOKED", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED}, #else {"SSLV3_ALERT_CERTIFICATE_REVOKED", 20, 1044}, #endif + #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED + {"SSLV3_ALERT_CERTIFICATE_REVOKED", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED}, + #else + {"SSLV3_ALERT_CERTIFICATE_REVOKED", 20, 1044}, + #endif + #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN + {"SSLV3_ALERT_CERTIFICATE_UNKNOWN", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN}, + #else + {"SSLV3_ALERT_CERTIFICATE_UNKNOWN", 20, 1046}, + #endif #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN {"SSLV3_ALERT_CERTIFICATE_UNKNOWN", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN}, #else @@ -7350,6 +7792,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"SSLV3_ALERT_DECOMPRESSION_FAILURE", 20, 1030}, #endif + #ifdef SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE + {"SSLV3_ALERT_DECOMPRESSION_FAILURE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE}, + #else + {"SSLV3_ALERT_DECOMPRESSION_FAILURE", 20, 1030}, + #endif + #ifdef SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE + {"SSLV3_ALERT_HANDSHAKE_FAILURE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE}, + #else + {"SSLV3_ALERT_HANDSHAKE_FAILURE", 20, 1040}, + #endif #ifdef SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE {"SSLV3_ALERT_HANDSHAKE_FAILURE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE}, #else @@ -7360,11 +7812,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"SSLV3_ALERT_ILLEGAL_PARAMETER", 20, 1047}, #endif + #ifdef SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER + {"SSLV3_ALERT_ILLEGAL_PARAMETER", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER}, + #else + {"SSLV3_ALERT_ILLEGAL_PARAMETER", 20, 1047}, + #endif #ifdef SSL_R_SSLV3_ALERT_NO_CERTIFICATE {"SSLV3_ALERT_NO_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_NO_CERTIFICATE}, #else {"SSLV3_ALERT_NO_CERTIFICATE", 20, 1041}, #endif + #ifdef SSL_R_SSLV3_ALERT_NO_CERTIFICATE + {"SSLV3_ALERT_NO_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_NO_CERTIFICATE}, + #else + {"SSLV3_ALERT_NO_CERTIFICATE", 20, 1041}, + #endif + #ifdef SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE + {"SSLV3_ALERT_UNEXPECTED_MESSAGE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE}, + #else + {"SSLV3_ALERT_UNEXPECTED_MESSAGE", 20, 1010}, + #endif #ifdef SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE {"SSLV3_ALERT_UNEXPECTED_MESSAGE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE}, #else @@ -7375,6 +7842,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", 20, 1043}, #endif + #ifdef SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE + {"SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE}, + #else + {"SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", 20, 1043}, + #endif #ifdef SSL_R_SSL_COMMAND_SECTION_EMPTY {"SSL_COMMAND_SECTION_EMPTY", ERR_LIB_SSL, SSL_R_SSL_COMMAND_SECTION_EMPTY}, #else @@ -7450,6 +7922,36 @@ static struct py_ssl_error_code error_codes[] = { #else {"STILL_IN_INIT", 20, 121}, #endif + #ifdef SSL_R_STREAM_COUNT_LIMITED + {"STREAM_COUNT_LIMITED", ERR_LIB_SSL, SSL_R_STREAM_COUNT_LIMITED}, + #else + {"STREAM_COUNT_LIMITED", 20, 411}, + #endif + #ifdef SSL_R_STREAM_FINISHED + {"STREAM_FINISHED", ERR_LIB_SSL, SSL_R_STREAM_FINISHED}, + #else + {"STREAM_FINISHED", 20, 365}, + #endif + #ifdef SSL_R_STREAM_RECV_ONLY + {"STREAM_RECV_ONLY", ERR_LIB_SSL, SSL_R_STREAM_RECV_ONLY}, + #else + {"STREAM_RECV_ONLY", 20, 366}, + #endif + #ifdef SSL_R_STREAM_RESET + {"STREAM_RESET", ERR_LIB_SSL, SSL_R_STREAM_RESET}, + #else + {"STREAM_RESET", 20, 375}, + #endif + #ifdef SSL_R_STREAM_SEND_ONLY + {"STREAM_SEND_ONLY", ERR_LIB_SSL, SSL_R_STREAM_SEND_ONLY}, + #else + {"STREAM_SEND_ONLY", 20, 379}, + #endif + #ifdef SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED + {"TLSV13_ALERT_CERTIFICATE_REQUIRED", ERR_LIB_SSL, SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED}, + #else + {"TLSV13_ALERT_CERTIFICATE_REQUIRED", 20, 1116}, + #endif #ifdef SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED {"TLSV13_ALERT_CERTIFICATE_REQUIRED", ERR_LIB_SSL, SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED}, #else @@ -7460,6 +7962,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV13_ALERT_MISSING_EXTENSION", 20, 1109}, #endif + #ifdef SSL_R_TLSV13_ALERT_MISSING_EXTENSION + {"TLSV13_ALERT_MISSING_EXTENSION", ERR_LIB_SSL, SSL_R_TLSV13_ALERT_MISSING_EXTENSION}, + #else + {"TLSV13_ALERT_MISSING_EXTENSION", 20, 1109}, + #endif + #ifdef SSL_R_TLSV1_ALERT_ACCESS_DENIED + {"TLSV1_ALERT_ACCESS_DENIED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_ACCESS_DENIED}, + #else + {"TLSV1_ALERT_ACCESS_DENIED", 20, 1049}, + #endif #ifdef SSL_R_TLSV1_ALERT_ACCESS_DENIED {"TLSV1_ALERT_ACCESS_DENIED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_ACCESS_DENIED}, #else @@ -7470,11 +7982,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_ALERT_DECODE_ERROR", 20, 1050}, #endif + #ifdef SSL_R_TLSV1_ALERT_DECODE_ERROR + {"TLSV1_ALERT_DECODE_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECODE_ERROR}, + #else + {"TLSV1_ALERT_DECODE_ERROR", 20, 1050}, + #endif #ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED {"TLSV1_ALERT_DECRYPTION_FAILED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPTION_FAILED}, #else {"TLSV1_ALERT_DECRYPTION_FAILED", 20, 1021}, #endif + #ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED + {"TLSV1_ALERT_DECRYPTION_FAILED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPTION_FAILED}, + #else + {"TLSV1_ALERT_DECRYPTION_FAILED", 20, 1021}, + #endif + #ifdef SSL_R_TLSV1_ALERT_DECRYPT_ERROR + {"TLSV1_ALERT_DECRYPT_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPT_ERROR}, + #else + {"TLSV1_ALERT_DECRYPT_ERROR", 20, 1051}, + #endif #ifdef SSL_R_TLSV1_ALERT_DECRYPT_ERROR {"TLSV1_ALERT_DECRYPT_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPT_ERROR}, #else @@ -7485,6 +8012,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_ALERT_EXPORT_RESTRICTION", 20, 1060}, #endif + #ifdef SSL_R_TLSV1_ALERT_EXPORT_RESTRICTION + {"TLSV1_ALERT_EXPORT_RESTRICTION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_EXPORT_RESTRICTION}, + #else + {"TLSV1_ALERT_EXPORT_RESTRICTION", 20, 1060}, + #endif + #ifdef SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK + {"TLSV1_ALERT_INAPPROPRIATE_FALLBACK", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK}, + #else + {"TLSV1_ALERT_INAPPROPRIATE_FALLBACK", 20, 1086}, + #endif #ifdef SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK {"TLSV1_ALERT_INAPPROPRIATE_FALLBACK", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK}, #else @@ -7495,11 +8032,36 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_ALERT_INSUFFICIENT_SECURITY", 20, 1071}, #endif + #ifdef SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY + {"TLSV1_ALERT_INSUFFICIENT_SECURITY", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY}, + #else + {"TLSV1_ALERT_INSUFFICIENT_SECURITY", 20, 1071}, + #endif + #ifdef SSL_R_TLSV1_ALERT_INTERNAL_ERROR + {"TLSV1_ALERT_INTERNAL_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INTERNAL_ERROR}, + #else + {"TLSV1_ALERT_INTERNAL_ERROR", 20, 1080}, + #endif #ifdef SSL_R_TLSV1_ALERT_INTERNAL_ERROR {"TLSV1_ALERT_INTERNAL_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INTERNAL_ERROR}, #else {"TLSV1_ALERT_INTERNAL_ERROR", 20, 1080}, #endif + #ifdef SSL_R_TLSV1_ALERT_NO_APPLICATION_PROTOCOL + {"TLSV1_ALERT_NO_APPLICATION_PROTOCOL", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_NO_APPLICATION_PROTOCOL}, + #else + {"TLSV1_ALERT_NO_APPLICATION_PROTOCOL", 20, 1120}, + #endif + #ifdef SSL_R_TLSV1_ALERT_NO_APPLICATION_PROTOCOL + {"TLSV1_ALERT_NO_APPLICATION_PROTOCOL", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_NO_APPLICATION_PROTOCOL}, + #else + {"TLSV1_ALERT_NO_APPLICATION_PROTOCOL", 20, 1120}, + #endif + #ifdef SSL_R_TLSV1_ALERT_NO_RENEGOTIATION + {"TLSV1_ALERT_NO_RENEGOTIATION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_NO_RENEGOTIATION}, + #else + {"TLSV1_ALERT_NO_RENEGOTIATION", 20, 1100}, + #endif #ifdef SSL_R_TLSV1_ALERT_NO_RENEGOTIATION {"TLSV1_ALERT_NO_RENEGOTIATION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_NO_RENEGOTIATION}, #else @@ -7510,6 +8072,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_ALERT_PROTOCOL_VERSION", 20, 1070}, #endif + #ifdef SSL_R_TLSV1_ALERT_PROTOCOL_VERSION + {"TLSV1_ALERT_PROTOCOL_VERSION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_PROTOCOL_VERSION}, + #else + {"TLSV1_ALERT_PROTOCOL_VERSION", 20, 1070}, + #endif + #ifdef SSL_R_TLSV1_ALERT_RECORD_OVERFLOW + {"TLSV1_ALERT_RECORD_OVERFLOW", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_RECORD_OVERFLOW}, + #else + {"TLSV1_ALERT_RECORD_OVERFLOW", 20, 1022}, + #endif #ifdef SSL_R_TLSV1_ALERT_RECORD_OVERFLOW {"TLSV1_ALERT_RECORD_OVERFLOW", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_RECORD_OVERFLOW}, #else @@ -7520,11 +8092,36 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_ALERT_UNKNOWN_CA", 20, 1048}, #endif + #ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA + {"TLSV1_ALERT_UNKNOWN_CA", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_UNKNOWN_CA}, + #else + {"TLSV1_ALERT_UNKNOWN_CA", 20, 1048}, + #endif + #ifdef SSL_R_TLSV1_ALERT_UNKNOWN_PSK_IDENTITY + {"TLSV1_ALERT_UNKNOWN_PSK_IDENTITY", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_UNKNOWN_PSK_IDENTITY}, + #else + {"TLSV1_ALERT_UNKNOWN_PSK_IDENTITY", 20, 1115}, + #endif + #ifdef SSL_R_TLSV1_ALERT_UNKNOWN_PSK_IDENTITY + {"TLSV1_ALERT_UNKNOWN_PSK_IDENTITY", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_UNKNOWN_PSK_IDENTITY}, + #else + {"TLSV1_ALERT_UNKNOWN_PSK_IDENTITY", 20, 1115}, + #endif #ifdef SSL_R_TLSV1_ALERT_USER_CANCELLED {"TLSV1_ALERT_USER_CANCELLED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_USER_CANCELLED}, #else {"TLSV1_ALERT_USER_CANCELLED", 20, 1090}, #endif + #ifdef SSL_R_TLSV1_ALERT_USER_CANCELLED + {"TLSV1_ALERT_USER_CANCELLED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_USER_CANCELLED}, + #else + {"TLSV1_ALERT_USER_CANCELLED", 20, 1090}, + #endif + #ifdef SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE + {"TLSV1_BAD_CERTIFICATE_HASH_VALUE", ERR_LIB_SSL, SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE}, + #else + {"TLSV1_BAD_CERTIFICATE_HASH_VALUE", 20, 1114}, + #endif #ifdef SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE {"TLSV1_BAD_CERTIFICATE_HASH_VALUE", ERR_LIB_SSL, SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE}, #else @@ -7535,11 +8132,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", 20, 1113}, #endif + #ifdef SSL_R_TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE + {"TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", ERR_LIB_SSL, SSL_R_TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE}, + #else + {"TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", 20, 1113}, + #endif #ifdef SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE {"TLSV1_CERTIFICATE_UNOBTAINABLE", ERR_LIB_SSL, SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE}, #else {"TLSV1_CERTIFICATE_UNOBTAINABLE", 20, 1111}, #endif + #ifdef SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE + {"TLSV1_CERTIFICATE_UNOBTAINABLE", ERR_LIB_SSL, SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE}, + #else + {"TLSV1_CERTIFICATE_UNOBTAINABLE", 20, 1111}, + #endif + #ifdef SSL_R_TLSV1_UNRECOGNIZED_NAME + {"TLSV1_UNRECOGNIZED_NAME", ERR_LIB_SSL, SSL_R_TLSV1_UNRECOGNIZED_NAME}, + #else + {"TLSV1_UNRECOGNIZED_NAME", 20, 1112}, + #endif #ifdef SSL_R_TLSV1_UNRECOGNIZED_NAME {"TLSV1_UNRECOGNIZED_NAME", ERR_LIB_SSL, SSL_R_TLSV1_UNRECOGNIZED_NAME}, #else @@ -7550,6 +8162,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_UNSUPPORTED_EXTENSION", 20, 1110}, #endif + #ifdef SSL_R_TLSV1_UNSUPPORTED_EXTENSION + {"TLSV1_UNSUPPORTED_EXTENSION", ERR_LIB_SSL, SSL_R_TLSV1_UNSUPPORTED_EXTENSION}, + #else + {"TLSV1_UNSUPPORTED_EXTENSION", 20, 1110}, + #endif #ifdef SSL_R_TLS_ILLEGAL_EXPORTER_LABEL {"TLS_ILLEGAL_EXPORTER_LABEL", ERR_LIB_SSL, SSL_R_TLS_ILLEGAL_EXPORTER_LABEL}, #else @@ -7665,6 +8282,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNKNOWN_KEY_EXCHANGE_TYPE", 20, 250}, #endif + #ifdef SSL_R_UNKNOWN_MANDATORY_PARAMETER + {"UNKNOWN_MANDATORY_PARAMETER", ERR_LIB_SSL, SSL_R_UNKNOWN_MANDATORY_PARAMETER}, + #else + {"UNKNOWN_MANDATORY_PARAMETER", 20, 323}, + #endif #ifdef SSL_R_UNKNOWN_PKEY_TYPE {"UNKNOWN_PKEY_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_PKEY_TYPE}, #else @@ -7700,6 +8322,21 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_COMPRESSION_ALGORITHM", 20, 257}, #endif + #ifdef SSL_R_UNSUPPORTED_CONFIG_VALUE + {"UNSUPPORTED_CONFIG_VALUE", ERR_LIB_SSL, SSL_R_UNSUPPORTED_CONFIG_VALUE}, + #else + {"UNSUPPORTED_CONFIG_VALUE", 20, 414}, + #endif + #ifdef SSL_R_UNSUPPORTED_CONFIG_VALUE_CLASS + {"UNSUPPORTED_CONFIG_VALUE_CLASS", ERR_LIB_SSL, SSL_R_UNSUPPORTED_CONFIG_VALUE_CLASS}, + #else + {"UNSUPPORTED_CONFIG_VALUE_CLASS", 20, 415}, + #endif + #ifdef SSL_R_UNSUPPORTED_CONFIG_VALUE_OP + {"UNSUPPORTED_CONFIG_VALUE_OP", ERR_LIB_SSL, SSL_R_UNSUPPORTED_CONFIG_VALUE_OP}, + #else + {"UNSUPPORTED_CONFIG_VALUE_OP", 20, 416}, + #endif #ifdef SSL_R_UNSUPPORTED_ELLIPTIC_CURVE {"UNSUPPORTED_ELLIPTIC_CURVE", ERR_LIB_SSL, SSL_R_UNSUPPORTED_ELLIPTIC_CURVE}, #else @@ -7720,6 +8357,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_STATUS_TYPE", 20, 329}, #endif + #ifdef SSL_R_UNSUPPORTED_WRITE_FLAG + {"UNSUPPORTED_WRITE_FLAG", ERR_LIB_SSL, SSL_R_UNSUPPORTED_WRITE_FLAG}, + #else + {"UNSUPPORTED_WRITE_FLAG", 20, 412}, + #endif #ifdef SSL_R_USE_SRTP_NOT_NEGOTIATED {"USE_SRTP_NOT_NEGOTIATED", ERR_LIB_SSL, SSL_R_USE_SRTP_NOT_NEGOTIATED}, #else @@ -7750,6 +8392,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"WRONG_CURVE", 20, 378}, #endif + #ifdef SSL_R_WRONG_RPK_TYPE + {"WRONG_RPK_TYPE", ERR_LIB_SSL, SSL_R_WRONG_RPK_TYPE}, + #else + {"WRONG_RPK_TYPE", 20, 351}, + #endif #ifdef SSL_R_WRONG_SIGNATURE_LENGTH {"WRONG_SIGNATURE_LENGTH", ERR_LIB_SSL, SSL_R_WRONG_SIGNATURE_LENGTH}, #else @@ -8055,6 +8702,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"BAD_OBJECT", 34, 119}, #endif + #ifdef X509V3_R_BAD_OPTION + {"BAD_OPTION", ERR_LIB_X509V3, X509V3_R_BAD_OPTION}, + #else + {"BAD_OPTION", 34, 170}, + #endif + #ifdef X509V3_R_BAD_VALUE + {"BAD_VALUE", ERR_LIB_X509V3, X509V3_R_BAD_VALUE}, + #else + {"BAD_VALUE", 34, 171}, + #endif #ifdef X509V3_R_BN_DEC2BN_ERROR {"BN_DEC2BN_ERROR", ERR_LIB_X509V3, X509V3_R_BN_DEC2BN_ERROR}, #else @@ -8370,6 +9027,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNKNOWN_OPTION", 34, 120}, #endif + #ifdef X509V3_R_UNKNOWN_VALUE + {"UNKNOWN_VALUE", ERR_LIB_X509V3, X509V3_R_UNKNOWN_VALUE}, + #else + {"UNKNOWN_VALUE", 34, 172}, + #endif #ifdef X509V3_R_UNSUPPORTED_OPTION {"UNSUPPORTED_OPTION", ERR_LIB_X509V3, X509V3_R_UNSUPPORTED_OPTION}, #else @@ -8430,6 +9092,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"CRL_VERIFY_FAILURE", 11, 131}, #endif + #ifdef X509_R_DUPLICATE_ATTRIBUTE + {"DUPLICATE_ATTRIBUTE", ERR_LIB_X509, X509_R_DUPLICATE_ATTRIBUTE}, + #else + {"DUPLICATE_ATTRIBUTE", 11, 140}, + #endif #ifdef X509_R_ERROR_GETTING_MD_BY_NID {"ERROR_GETTING_MD_BY_NID", ERR_LIB_X509, X509_R_ERROR_GETTING_MD_BY_NID}, #else @@ -8590,6 +9257,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_ALGORITHM", 11, 111}, #endif + #ifdef X509_R_UNSUPPORTED_VERSION + {"UNSUPPORTED_VERSION", ERR_LIB_X509, X509_R_UNSUPPORTED_VERSION}, + #else + {"UNSUPPORTED_VERSION", 11, 145}, + #endif #ifdef X509_R_WRONG_LOOKUP_TYPE {"WRONG_LOOKUP_TYPE", ERR_LIB_X509, X509_R_WRONG_LOOKUP_TYPE}, #else diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 21be53e78841d5..a08b32fa45db3e 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -70,9 +70,7 @@ def clean_lines(text): Python/thread_pthread_stubs.h # only huge constants (safe but parsing is slow) -Modules/_ssl_data_31.h -Modules/_ssl_data_300.h -Modules/_ssl_data_111.h +Modules/_ssl_data_*.h Modules/cjkcodecs/mappings_*.h Modules/unicodedata_db.h Modules/unicodename_db.h diff --git a/Tools/ssl/make_ssl_data.py b/Tools/ssl/make_ssl_data.py index d24e02210d489c..da05d2bc8b9752 100755 --- a/Tools/ssl/make_ssl_data.py +++ b/Tools/ssl/make_ssl_data.py @@ -5,9 +5,28 @@ `library` and `reason` mnemonics to a more recent OpenSSL version. It takes two arguments: -- the path to the OpenSSL source tree (e.g. git checkout) +- the path to the OpenSSL git checkout - the path to the header file to be generated Modules/_ssl_data_{version}.h - error codes are version specific + +The OpenSSL git checkout should be at a specific tag, using commands like: + git tag --list 'openssl-*' + git switch --detach openssl-3.4.0 + + +After generating the definitions, compare the result with newest pre-existing file. +You can use a command like: + + git diff --no-index Modules/_ssl_data_31.h Modules/_ssl_data_34.h + +- If the new version *only* adds new definitions, remove the pre-existing file + and adjust the #include in _ssl.c to point to the new version. +- If the new version removes or renumbers some definitions, keep both files and + add a new #include in _ssl.c. + +A newly supported OpenSSL version should also be added to: +- Tools/ssl/multissltests.py +- .github/workflows/build.yml """ import argparse @@ -15,6 +34,7 @@ import operator import os import re +import subprocess parser = argparse.ArgumentParser( @@ -117,9 +137,17 @@ def main(): # sort by libname, numeric error code args.reasons = sorted(reasons, key=operator.itemgetter(0, 3)) + git_describe = subprocess.run( + ['git', 'describe', '--long', '--dirty'], + cwd=args.srcdir, + capture_output=True, + encoding='utf-8', + check=True, + ) lines = [ - "/* File generated by Tools/ssl/make_ssl_data.py */" - f"/* Generated on {datetime.datetime.utcnow().isoformat()} */" + "/* File generated by Tools/ssl/make_ssl_data.py */", + f"/* Generated on {datetime.datetime.now(datetime.UTC).isoformat()} */", + f"/* Generated from Git commit {git_describe.stdout.strip()} */", ] lines.extend(gen_library_codes(args)) lines.append("") diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index eae0e0c5e8761f..2cd0c39b5a6477 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -51,6 +51,8 @@ "3.1.7", "3.2.3", "3.3.2", + "3.4.0", + # See make_ssl_data.py for notes on adding a new version. ] LIBRESSL_OLD_VERSIONS = [ From 49fee592a4fad17781bb4a78f95085d6edbb24d5 Mon Sep 17 00:00:00 2001 From: Jun Komoda <45822440+junkmd@users.noreply.github.com> Date: Fri, 29 Nov 2024 00:59:47 +0900 Subject: [PATCH 041/427] Touch up docs for ctypes.FormatError & WinError (GH-127210) Reformat paragraphs, add backquotes, and directives. --- Doc/library/ctypes.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index f490f7563b58a5..bd9529db9ee65a 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2035,9 +2035,9 @@ Utility functions .. function:: FormatError([code]) - Returns a textual description of the error code *code*. If no - error code is specified, the last error code is used by calling the Windows - api function GetLastError. + Returns a textual description of the error code *code*. If no error code is + specified, the last error code is used by calling the Windows API function + :func:`GetLastError`. .. availability:: Windows @@ -2142,9 +2142,8 @@ Utility functions .. function:: WinError(code=None, descr=None) - This function is probably the worst-named thing in ctypes. It - creates an instance of :exc:`OSError`. If *code* is not specified, - ``GetLastError`` is called to determine the error code. If *descr* is not + Creates an instance of :exc:`OSError`. If *code* is not specified, + :func:`GetLastError` is called to determine the error code. If *descr* is not specified, :func:`FormatError` is called to get a textual description of the error. From 20657fbdb14d50ca4ec115da0cbef155871d8d33 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 28 Nov 2024 17:35:48 +0100 Subject: [PATCH 042/427] gh-127190: Fix local_setattro() error handling (#127366) Don't make the assumption that the 'name' argument is a string. Use repr() to format the 'name' argument instead. --- Lib/test/test_threading_local.py | 15 +++++++++++++++ Modules/_threadmodule.c | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_threading_local.py b/Lib/test/test_threading_local.py index f0b829a978feb5..3a58afd8194a32 100644 --- a/Lib/test/test_threading_local.py +++ b/Lib/test/test_threading_local.py @@ -208,6 +208,21 @@ def test_threading_local_clear_race(self): _testcapi.join_temporary_c_thread() + @support.cpython_only + def test_error(self): + class Loop(self._local): + attr = 1 + + # Trick the "if name == '__dict__':" test of __setattr__() + # to always be true + class NameCompareTrue: + def __eq__(self, other): + return True + + loop = Loop() + with self.assertRaisesRegex(AttributeError, 'Loop.*read-only'): + loop.__setattr__(NameCompareTrue(), 2) + class ThreadLocalTest(unittest.TestCase, BaseLocalTest): _local = _thread._local diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index f2a420ac1c589d..4a45445e2f62db 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1624,7 +1624,7 @@ local_setattro(localobject *self, PyObject *name, PyObject *v) } if (r == 1) { PyErr_Format(PyExc_AttributeError, - "'%.100s' object attribute '%U' is read-only", + "'%.100s' object attribute %R is read-only", Py_TYPE(self)->tp_name, name); goto err; } From b83be9c9718aac42d0d8fc689a829d6594192afa Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 28 Nov 2024 19:03:09 +0200 Subject: [PATCH 043/427] gh-127359: Pin Tcl/Tk to 8 (8.6) for testing macOS (#127365) --- .github/workflows/reusable-macos.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 915481d0737c7d..4c3dd10194f8cb 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -37,7 +37,10 @@ jobs: path: config.cache key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk make + run: | + brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make + # Because alternate versions are not symlinked into place by default: + brew link tcl-tk@8 - name: Configure CPython run: | GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ From dd3a87d2a8f8750978359a99de2c5cb2168351d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D1=8C=D1=8F=20=D0=9B=D1=8E=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D1=81=D0=BA=D0=B8=D0=B9?= <100635212+lubaskinc0de@users.noreply.github.com> Date: Fri, 29 Nov 2024 12:00:50 +0300 Subject: [PATCH 044/427] gh-127303: Add docs for token.EXACT_TOKEN_TYPES (#127304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --------- Co-authored-by: Terry Jan Reedy Co-authored-by: Tomas R. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/token.rst | 7 +++++++ Lib/token.py | 3 ++- Misc/ACKS | 1 + .../Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst | 1 + Tools/build/generate_token.py | 3 ++- 5 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 0cc9dddd91ed6b..40982f32b4beee 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -79,6 +79,13 @@ the :mod:`tokenize` module. ``type_comments=True``. +.. data:: EXACT_TOKEN_TYPES + + A dictionary mapping the string representation of a token to its numeric code. + + .. versionadded:: 3.8 + + .. versionchanged:: 3.5 Added :data:`!AWAIT` and :data:`!ASYNC` tokens. diff --git a/Lib/token.py b/Lib/token.py index b620317106e173..54d7cdccadc79a 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,8 @@ """Token constants.""" # Auto-generated by Tools/build/generate_token.py -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] ENDMARKER = 0 NAME = 1 diff --git a/Misc/ACKS b/Misc/ACKS index 08cd293eac3835..cd34846574b304 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1154,6 +1154,7 @@ Mark Lutz Taras Lyapun Jim Lynch Mikael Lyngvig +Ilya Lyubavski Jeff MacDonald John Machin Andrew I MacIntyre diff --git a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst new file mode 100644 index 00000000000000..58ebf5d0abe141 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst @@ -0,0 +1 @@ +Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`. diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 16c38841e44a4d..d32747f19945d8 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -226,7 +226,8 @@ def make_rst(infile, outfile='Doc/library/token-list.inc'): # {AUTO_GENERATED_BY_SCRIPT} ''' token_py_template += ''' -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] %s N_TOKENS = %d From 762c603a866146afc7db2591fb49605e0858e9b1 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 29 Nov 2024 15:17:16 +0530 Subject: [PATCH 045/427] gh-126881: fix finalization of dtoa state (#126904) --- .../2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst | 1 + Python/pylifecycle.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst new file mode 100644 index 00000000000000..13381c7630d7ce --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst @@ -0,0 +1 @@ +Fix crash in finalization of dtoa state. Patch by Kumar Aditya. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 23882d083844ac..ceb30e9f02df2c 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1888,7 +1888,6 @@ finalize_interp_clear(PyThreadState *tstate) _PyXI_Fini(tstate->interp); _PyExc_ClearExceptionGroupType(tstate->interp); _Py_clear_generic_types(tstate->interp); - _PyDtoa_Fini(tstate->interp); /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); @@ -1910,6 +1909,9 @@ finalize_interp_clear(PyThreadState *tstate) finalize_interp_types(tstate->interp); + /* Finalize dtoa at last so that finalizers calling repr of float doesn't crash */ + _PyDtoa_Fini(tstate->interp); + /* Free any delayed free requests immediately */ _PyMem_FiniDelayed(tstate->interp); From 99490913a08adcf2fe5e69b82772a829ec462275 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 29 Nov 2024 05:12:13 -0500 Subject: [PATCH 046/427] gh-127341: Argument Clinic: fix compiler warnings for getters with docstrings (#127310) Co-authored-by: Erlend E. Aasland --- Lib/test/clinic.test.c | 12 +-- Modules/_io/clinic/bufferedio.c.h | 14 +-- Modules/_io/clinic/stringio.c.h | 14 +-- Modules/_io/clinic/textio.c.h | 44 +++----- Modules/clinic/_ssl.c.h | 152 +++++++-------------------- Tools/clinic/libclinic/parse_args.py | 10 +- 6 files changed, 68 insertions(+), 178 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 4ad0b8b0910bbe..b6ae04ecf2f8ed 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5303,9 +5303,7 @@ Test_meth_coexist_impl(TestObj *self) Test.property [clinic start generated code]*/ -#if defined(Test_property_HAS_DOCSTR) -# define Test_property_DOCSTR Test_property__doc__ -#else +#if !defined(Test_property_DOCSTR) # define Test_property_DOCSTR NULL #endif #if defined(TEST_PROPERTY_GETSETDEF) @@ -5326,16 +5324,14 @@ Test_property_get(TestObj *self, void *Py_UNUSED(context)) static PyObject * Test_property_get_impl(TestObj *self) -/*[clinic end generated code: output=27b519719d992e03 input=2d92b3449fbc7d2b]*/ +/*[clinic end generated code: output=7cadd0f539805266 input=2d92b3449fbc7d2b]*/ /*[clinic input] @setter Test.property [clinic start generated code]*/ -#if defined(TEST_PROPERTY_HAS_DOCSTR) -# define Test_property_DOCSTR Test_property__doc__ -#else +#if !defined(Test_property_DOCSTR) # define Test_property_DOCSTR NULL #endif #if defined(TEST_PROPERTY_GETSETDEF) @@ -5360,7 +5356,7 @@ Test_property_set(TestObj *self, PyObject *value, void *Py_UNUSED(context)) static int Test_property_set_impl(TestObj *self, PyObject *value) -/*[clinic end generated code: output=d51023f17c4ac3a1 input=3bc3f46a23c83a88]*/ +/*[clinic end generated code: output=e4342fe9bb1d7817 input=3bc3f46a23c83a88]*/ /*[clinic input] output push diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index b703d7e6855398..e035bd99baca5f 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -330,9 +330,7 @@ _io__Buffered_simple_flush(buffered *self, PyObject *Py_UNUSED(ignored)) return return_value; } -#if defined(_io__Buffered_closed_HAS_DOCSTR) -# define _io__Buffered_closed_DOCSTR _io__Buffered_closed__doc__ -#else +#if !defined(_io__Buffered_closed_DOCSTR) # define _io__Buffered_closed_DOCSTR NULL #endif #if defined(_IO__BUFFERED_CLOSED_GETSETDEF) @@ -472,9 +470,7 @@ _io__Buffered_writable(buffered *self, PyObject *Py_UNUSED(ignored)) return return_value; } -#if defined(_io__Buffered_name_HAS_DOCSTR) -# define _io__Buffered_name_DOCSTR _io__Buffered_name__doc__ -#else +#if !defined(_io__Buffered_name_DOCSTR) # define _io__Buffered_name_DOCSTR NULL #endif #if defined(_IO__BUFFERED_NAME_GETSETDEF) @@ -499,9 +495,7 @@ _io__Buffered_name_get(buffered *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io__Buffered_mode_HAS_DOCSTR) -# define _io__Buffered_mode_DOCSTR _io__Buffered_mode__doc__ -#else +#if !defined(_io__Buffered_mode_DOCSTR) # define _io__Buffered_mode_DOCSTR NULL #endif #if defined(_IO__BUFFERED_MODE_GETSETDEF) @@ -1252,4 +1246,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=36abca5bd2f63ea1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8f28a97987a9fbe1 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index ac8b4b7b85b76d..6f9205af32f010 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -476,9 +476,7 @@ _io_StringIO___setstate__(stringio *self, PyObject *state) return return_value; } -#if defined(_io_StringIO_closed_HAS_DOCSTR) -# define _io_StringIO_closed_DOCSTR _io_StringIO_closed__doc__ -#else +#if !defined(_io_StringIO_closed_DOCSTR) # define _io_StringIO_closed_DOCSTR NULL #endif #if defined(_IO_STRINGIO_CLOSED_GETSETDEF) @@ -503,9 +501,7 @@ _io_StringIO_closed_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_StringIO_line_buffering_HAS_DOCSTR) -# define _io_StringIO_line_buffering_DOCSTR _io_StringIO_line_buffering__doc__ -#else +#if !defined(_io_StringIO_line_buffering_DOCSTR) # define _io_StringIO_line_buffering_DOCSTR NULL #endif #if defined(_IO_STRINGIO_LINE_BUFFERING_GETSETDEF) @@ -530,9 +526,7 @@ _io_StringIO_line_buffering_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_StringIO_newlines_HAS_DOCSTR) -# define _io_StringIO_newlines_DOCSTR _io_StringIO_newlines__doc__ -#else +#if !defined(_io_StringIO_newlines_DOCSTR) # define _io_StringIO_newlines_DOCSTR NULL #endif #if defined(_IO_STRINGIO_NEWLINES_GETSETDEF) @@ -556,4 +550,4 @@ _io_StringIO_newlines_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=8c8d4f8fa32986bb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d2b092274469d42 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index c9301c5a23fa86..160f80ada43660 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -208,11 +208,9 @@ PyDoc_STRVAR(_io__TextIOBase_encoding__doc__, "Encoding of the text stream.\n" "\n" "Subclasses should override."); -#define _io__TextIOBase_encoding_HAS_DOCSTR +#define _io__TextIOBase_encoding_DOCSTR _io__TextIOBase_encoding__doc__ -#if defined(_io__TextIOBase_encoding_HAS_DOCSTR) -# define _io__TextIOBase_encoding_DOCSTR _io__TextIOBase_encoding__doc__ -#else +#if !defined(_io__TextIOBase_encoding_DOCSTR) # define _io__TextIOBase_encoding_DOCSTR NULL #endif #if defined(_IO__TEXTIOBASE_ENCODING_GETSETDEF) @@ -237,11 +235,9 @@ PyDoc_STRVAR(_io__TextIOBase_newlines__doc__, "Only line endings translated during reading are considered.\n" "\n" "Subclasses should override."); -#define _io__TextIOBase_newlines_HAS_DOCSTR +#define _io__TextIOBase_newlines_DOCSTR _io__TextIOBase_newlines__doc__ -#if defined(_io__TextIOBase_newlines_HAS_DOCSTR) -# define _io__TextIOBase_newlines_DOCSTR _io__TextIOBase_newlines__doc__ -#else +#if !defined(_io__TextIOBase_newlines_DOCSTR) # define _io__TextIOBase_newlines_DOCSTR NULL #endif #if defined(_IO__TEXTIOBASE_NEWLINES_GETSETDEF) @@ -264,11 +260,9 @@ PyDoc_STRVAR(_io__TextIOBase_errors__doc__, "The error setting of the decoder or encoder.\n" "\n" "Subclasses should override."); -#define _io__TextIOBase_errors_HAS_DOCSTR +#define _io__TextIOBase_errors_DOCSTR _io__TextIOBase_errors__doc__ -#if defined(_io__TextIOBase_errors_HAS_DOCSTR) -# define _io__TextIOBase_errors_DOCSTR _io__TextIOBase_errors__doc__ -#else +#if !defined(_io__TextIOBase_errors_DOCSTR) # define _io__TextIOBase_errors_DOCSTR NULL #endif #if defined(_IO__TEXTIOBASE_ERRORS_GETSETDEF) @@ -1138,9 +1132,7 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) return return_value; } -#if defined(_io_TextIOWrapper_name_HAS_DOCSTR) -# define _io_TextIOWrapper_name_DOCSTR _io_TextIOWrapper_name__doc__ -#else +#if !defined(_io_TextIOWrapper_name_DOCSTR) # define _io_TextIOWrapper_name_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_NAME_GETSETDEF) @@ -1165,9 +1157,7 @@ _io_TextIOWrapper_name_get(textio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_TextIOWrapper_closed_HAS_DOCSTR) -# define _io_TextIOWrapper_closed_DOCSTR _io_TextIOWrapper_closed__doc__ -#else +#if !defined(_io_TextIOWrapper_closed_DOCSTR) # define _io_TextIOWrapper_closed_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_CLOSED_GETSETDEF) @@ -1192,9 +1182,7 @@ _io_TextIOWrapper_closed_get(textio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_TextIOWrapper_newlines_HAS_DOCSTR) -# define _io_TextIOWrapper_newlines_DOCSTR _io_TextIOWrapper_newlines__doc__ -#else +#if !defined(_io_TextIOWrapper_newlines_DOCSTR) # define _io_TextIOWrapper_newlines_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF) @@ -1219,9 +1207,7 @@ _io_TextIOWrapper_newlines_get(textio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_TextIOWrapper_errors_HAS_DOCSTR) -# define _io_TextIOWrapper_errors_DOCSTR _io_TextIOWrapper_errors__doc__ -#else +#if !defined(_io_TextIOWrapper_errors_DOCSTR) # define _io_TextIOWrapper_errors_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_ERRORS_GETSETDEF) @@ -1246,9 +1232,7 @@ _io_TextIOWrapper_errors_get(textio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_TextIOWrapper__CHUNK_SIZE_HAS_DOCSTR) -# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR _io_TextIOWrapper__CHUNK_SIZE__doc__ -#else +#if !defined(_io_TextIOWrapper__CHUNK_SIZE_DOCSTR) # define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) @@ -1273,9 +1257,7 @@ _io_TextIOWrapper__CHUNK_SIZE_get(textio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_HAS_DOCSTR) -# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR _io_TextIOWrapper__CHUNK_SIZE__doc__ -#else +#if !defined(_io_TextIOWrapper__CHUNK_SIZE_DOCSTR) # define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) @@ -1299,4 +1281,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=459c0e50acd772b1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1172c500a022c65d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 582eef16c13244..957f5ced3a2cee 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -258,9 +258,7 @@ _ssl__SSLSocket_compression(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) return _ssl__SSLSocket_compression_impl(self); } -#if defined(_ssl__SSLSocket_context_HAS_DOCSTR) -# define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ -#else +#if !defined(_ssl__SSLSocket_context_DOCSTR) # define _ssl__SSLSocket_context_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_CONTEXT_GETSETDEF) @@ -285,9 +283,7 @@ _ssl__SSLSocket_context_get(PySSLSocket *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLSOCKET_CONTEXT_HAS_DOCSTR) -# define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ -#else +#if !defined(_ssl__SSLSocket_context_DOCSTR) # define _ssl__SSLSocket_context_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_CONTEXT_GETSETDEF) @@ -314,11 +310,9 @@ _ssl__SSLSocket_context_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED( PyDoc_STRVAR(_ssl__SSLSocket_server_side__doc__, "Whether this is a server-side socket."); -#define _ssl__SSLSocket_server_side_HAS_DOCSTR +#define _ssl__SSLSocket_server_side_DOCSTR _ssl__SSLSocket_server_side__doc__ -#if defined(_ssl__SSLSocket_server_side_HAS_DOCSTR) -# define _ssl__SSLSocket_server_side_DOCSTR _ssl__SSLSocket_server_side__doc__ -#else +#if !defined(_ssl__SSLSocket_server_side_DOCSTR) # define _ssl__SSLSocket_server_side_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF) @@ -345,11 +339,9 @@ _ssl__SSLSocket_server_side_get(PySSLSocket *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl__SSLSocket_server_hostname__doc__, "The currently set server hostname (for SNI)."); -#define _ssl__SSLSocket_server_hostname_HAS_DOCSTR +#define _ssl__SSLSocket_server_hostname_DOCSTR _ssl__SSLSocket_server_hostname__doc__ -#if defined(_ssl__SSLSocket_server_hostname_HAS_DOCSTR) -# define _ssl__SSLSocket_server_hostname_DOCSTR _ssl__SSLSocket_server_hostname__doc__ -#else +#if !defined(_ssl__SSLSocket_server_hostname_DOCSTR) # define _ssl__SSLSocket_server_hostname_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF) @@ -374,9 +366,7 @@ _ssl__SSLSocket_server_hostname_get(PySSLSocket *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_ssl__SSLSocket_owner_HAS_DOCSTR) -# define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ -#else +#if !defined(_ssl__SSLSocket_owner_DOCSTR) # define _ssl__SSLSocket_owner_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_OWNER_GETSETDEF) @@ -401,9 +391,7 @@ _ssl__SSLSocket_owner_get(PySSLSocket *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLSOCKET_OWNER_HAS_DOCSTR) -# define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ -#else +#if !defined(_ssl__SSLSocket_owner_DOCSTR) # define _ssl__SSLSocket_owner_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_OWNER_GETSETDEF) @@ -664,9 +652,7 @@ _ssl__SSLSocket_verify_client_post_handshake(PySSLSocket *self, PyObject *Py_UNU return return_value; } -#if defined(_ssl__SSLSocket_session_HAS_DOCSTR) -# define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ -#else +#if !defined(_ssl__SSLSocket_session_DOCSTR) # define _ssl__SSLSocket_session_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_SESSION_GETSETDEF) @@ -691,9 +677,7 @@ _ssl__SSLSocket_session_get(PySSLSocket *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLSOCKET_SESSION_HAS_DOCSTR) -# define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ -#else +#if !defined(_ssl__SSLSocket_session_DOCSTR) # define _ssl__SSLSocket_session_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_SESSION_GETSETDEF) @@ -720,11 +704,9 @@ _ssl__SSLSocket_session_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED( PyDoc_STRVAR(_ssl__SSLSocket_session_reused__doc__, "Was the client session reused during handshake?"); -#define _ssl__SSLSocket_session_reused_HAS_DOCSTR +#define _ssl__SSLSocket_session_reused_DOCSTR _ssl__SSLSocket_session_reused__doc__ -#if defined(_ssl__SSLSocket_session_reused_HAS_DOCSTR) -# define _ssl__SSLSocket_session_reused_DOCSTR _ssl__SSLSocket_session_reused__doc__ -#else +#if !defined(_ssl__SSLSocket_session_reused_DOCSTR) # define _ssl__SSLSocket_session_reused_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF) @@ -873,9 +855,7 @@ _ssl__SSLContext__set_alpn_protocols(PySSLContext *self, PyObject *arg) return return_value; } -#if defined(_ssl__SSLContext_verify_mode_HAS_DOCSTR) -# define _ssl__SSLContext_verify_mode_DOCSTR _ssl__SSLContext_verify_mode__doc__ -#else +#if !defined(_ssl__SSLContext_verify_mode_DOCSTR) # define _ssl__SSLContext_verify_mode_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF) @@ -900,9 +880,7 @@ _ssl__SSLContext_verify_mode_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT_VERIFY_MODE_HAS_DOCSTR) -# define _ssl__SSLContext_verify_mode_DOCSTR _ssl__SSLContext_verify_mode__doc__ -#else +#if !defined(_ssl__SSLContext_verify_mode_DOCSTR) # define _ssl__SSLContext_verify_mode_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF) @@ -927,9 +905,7 @@ _ssl__SSLContext_verify_mode_set(PySSLContext *self, PyObject *value, void *Py_U return return_value; } -#if defined(_ssl__SSLContext_verify_flags_HAS_DOCSTR) -# define _ssl__SSLContext_verify_flags_DOCSTR _ssl__SSLContext_verify_flags__doc__ -#else +#if !defined(_ssl__SSLContext_verify_flags_DOCSTR) # define _ssl__SSLContext_verify_flags_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF) @@ -954,9 +930,7 @@ _ssl__SSLContext_verify_flags_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_HAS_DOCSTR) -# define _ssl__SSLContext_verify_flags_DOCSTR _ssl__SSLContext_verify_flags__doc__ -#else +#if !defined(_ssl__SSLContext_verify_flags_DOCSTR) # define _ssl__SSLContext_verify_flags_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF) @@ -981,9 +955,7 @@ _ssl__SSLContext_verify_flags_set(PySSLContext *self, PyObject *value, void *Py_ return return_value; } -#if defined(_ssl__SSLContext_minimum_version_HAS_DOCSTR) -# define _ssl__SSLContext_minimum_version_DOCSTR _ssl__SSLContext_minimum_version__doc__ -#else +#if !defined(_ssl__SSLContext_minimum_version_DOCSTR) # define _ssl__SSLContext_minimum_version_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF) @@ -1008,9 +980,7 @@ _ssl__SSLContext_minimum_version_get(PySSLContext *self, void *Py_UNUSED(context return return_value; } -#if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_HAS_DOCSTR) -# define _ssl__SSLContext_minimum_version_DOCSTR _ssl__SSLContext_minimum_version__doc__ -#else +#if !defined(_ssl__SSLContext_minimum_version_DOCSTR) # define _ssl__SSLContext_minimum_version_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF) @@ -1036,9 +1006,7 @@ _ssl__SSLContext_minimum_version_set(PySSLContext *self, PyObject *value, void * return return_value; } -#if defined(_ssl__SSLContext_maximum_version_HAS_DOCSTR) -# define _ssl__SSLContext_maximum_version_DOCSTR _ssl__SSLContext_maximum_version__doc__ -#else +#if !defined(_ssl__SSLContext_maximum_version_DOCSTR) # define _ssl__SSLContext_maximum_version_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF) @@ -1063,9 +1031,7 @@ _ssl__SSLContext_maximum_version_get(PySSLContext *self, void *Py_UNUSED(context return return_value; } -#if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_HAS_DOCSTR) -# define _ssl__SSLContext_maximum_version_DOCSTR _ssl__SSLContext_maximum_version__doc__ -#else +#if !defined(_ssl__SSLContext_maximum_version_DOCSTR) # define _ssl__SSLContext_maximum_version_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF) @@ -1091,9 +1057,7 @@ _ssl__SSLContext_maximum_version_set(PySSLContext *self, PyObject *value, void * return return_value; } -#if defined(_ssl__SSLContext_num_tickets_HAS_DOCSTR) -# define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ -#else +#if !defined(_ssl__SSLContext_num_tickets_DOCSTR) # define _ssl__SSLContext_num_tickets_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF) @@ -1118,9 +1082,7 @@ _ssl__SSLContext_num_tickets_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT_NUM_TICKETS_HAS_DOCSTR) -# define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ -#else +#if !defined(_ssl__SSLContext_num_tickets_DOCSTR) # define _ssl__SSLContext_num_tickets_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF) @@ -1145,9 +1107,7 @@ _ssl__SSLContext_num_tickets_set(PySSLContext *self, PyObject *value, void *Py_U return return_value; } -#if defined(_ssl__SSLContext_security_level_HAS_DOCSTR) -# define _ssl__SSLContext_security_level_DOCSTR _ssl__SSLContext_security_level__doc__ -#else +#if !defined(_ssl__SSLContext_security_level_DOCSTR) # define _ssl__SSLContext_security_level_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF) @@ -1172,9 +1132,7 @@ _ssl__SSLContext_security_level_get(PySSLContext *self, void *Py_UNUSED(context) return return_value; } -#if defined(_ssl__SSLContext_options_HAS_DOCSTR) -# define _ssl__SSLContext_options_DOCSTR _ssl__SSLContext_options__doc__ -#else +#if !defined(_ssl__SSLContext_options_DOCSTR) # define _ssl__SSLContext_options_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_OPTIONS_GETSETDEF) @@ -1199,9 +1157,7 @@ _ssl__SSLContext_options_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT_OPTIONS_HAS_DOCSTR) -# define _ssl__SSLContext_options_DOCSTR _ssl__SSLContext_options__doc__ -#else +#if !defined(_ssl__SSLContext_options_DOCSTR) # define _ssl__SSLContext_options_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_OPTIONS_GETSETDEF) @@ -1226,9 +1182,7 @@ _ssl__SSLContext_options_set(PySSLContext *self, PyObject *value, void *Py_UNUSE return return_value; } -#if defined(_ssl__SSLContext__host_flags_HAS_DOCSTR) -# define _ssl__SSLContext__host_flags_DOCSTR _ssl__SSLContext__host_flags__doc__ -#else +#if !defined(_ssl__SSLContext__host_flags_DOCSTR) # define _ssl__SSLContext__host_flags_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF) @@ -1253,9 +1207,7 @@ _ssl__SSLContext__host_flags_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT__HOST_FLAGS_HAS_DOCSTR) -# define _ssl__SSLContext__host_flags_DOCSTR _ssl__SSLContext__host_flags__doc__ -#else +#if !defined(_ssl__SSLContext__host_flags_DOCSTR) # define _ssl__SSLContext__host_flags_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF) @@ -1280,9 +1232,7 @@ _ssl__SSLContext__host_flags_set(PySSLContext *self, PyObject *value, void *Py_U return return_value; } -#if defined(_ssl__SSLContext_check_hostname_HAS_DOCSTR) -# define _ssl__SSLContext_check_hostname_DOCSTR _ssl__SSLContext_check_hostname__doc__ -#else +#if !defined(_ssl__SSLContext_check_hostname_DOCSTR) # define _ssl__SSLContext_check_hostname_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF) @@ -1307,9 +1257,7 @@ _ssl__SSLContext_check_hostname_get(PySSLContext *self, void *Py_UNUSED(context) return return_value; } -#if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_HAS_DOCSTR) -# define _ssl__SSLContext_check_hostname_DOCSTR _ssl__SSLContext_check_hostname__doc__ -#else +#if !defined(_ssl__SSLContext_check_hostname_DOCSTR) # define _ssl__SSLContext_check_hostname_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF) @@ -1334,9 +1282,7 @@ _ssl__SSLContext_check_hostname_set(PySSLContext *self, PyObject *value, void *P return return_value; } -#if defined(_ssl__SSLContext_protocol_HAS_DOCSTR) -# define _ssl__SSLContext_protocol_DOCSTR _ssl__SSLContext_protocol__doc__ -#else +#if !defined(_ssl__SSLContext_protocol_DOCSTR) # define _ssl__SSLContext_protocol_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_PROTOCOL_GETSETDEF) @@ -1799,9 +1745,7 @@ _ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name) return return_value; } -#if defined(_ssl__SSLContext_sni_callback_HAS_DOCSTR) -# define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ -#else +#if !defined(_ssl__SSLContext_sni_callback_DOCSTR) # define _ssl__SSLContext_sni_callback_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF) @@ -1826,9 +1770,7 @@ _ssl__SSLContext_sni_callback_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_HAS_DOCSTR) -# define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ -#else +#if !defined(_ssl__SSLContext_sni_callback_DOCSTR) # define _ssl__SSLContext_sni_callback_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF) @@ -2121,9 +2063,7 @@ _ssl_MemoryBIO(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } -#if defined(_ssl_MemoryBIO_pending_HAS_DOCSTR) -# define _ssl_MemoryBIO_pending_DOCSTR _ssl_MemoryBIO_pending__doc__ -#else +#if !defined(_ssl_MemoryBIO_pending_DOCSTR) # define _ssl_MemoryBIO_pending_DOCSTR NULL #endif #if defined(_SSL_MEMORYBIO_PENDING_GETSETDEF) @@ -2148,9 +2088,7 @@ _ssl_MemoryBIO_pending_get(PySSLMemoryBIO *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_ssl_MemoryBIO_eof_HAS_DOCSTR) -# define _ssl_MemoryBIO_eof_DOCSTR _ssl_MemoryBIO_eof__doc__ -#else +#if !defined(_ssl_MemoryBIO_eof_DOCSTR) # define _ssl_MemoryBIO_eof_DOCSTR NULL #endif #if defined(_SSL_MEMORYBIO_EOF_GETSETDEF) @@ -2279,9 +2217,7 @@ _ssl_MemoryBIO_write_eof(PySSLMemoryBIO *self, PyObject *Py_UNUSED(ignored)) return return_value; } -#if defined(_ssl_SSLSession_time_HAS_DOCSTR) -# define _ssl_SSLSession_time_DOCSTR _ssl_SSLSession_time__doc__ -#else +#if !defined(_ssl_SSLSession_time_DOCSTR) # define _ssl_SSLSession_time_DOCSTR NULL #endif #if defined(_SSL_SSLSESSION_TIME_GETSETDEF) @@ -2306,9 +2242,7 @@ _ssl_SSLSession_time_get(PySSLSession *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_ssl_SSLSession_timeout_HAS_DOCSTR) -# define _ssl_SSLSession_timeout_DOCSTR _ssl_SSLSession_timeout__doc__ -#else +#if !defined(_ssl_SSLSession_timeout_DOCSTR) # define _ssl_SSLSession_timeout_DOCSTR NULL #endif #if defined(_SSL_SSLSESSION_TIMEOUT_GETSETDEF) @@ -2333,9 +2267,7 @@ _ssl_SSLSession_timeout_get(PySSLSession *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_ssl_SSLSession_ticket_lifetime_hint_HAS_DOCSTR) -# define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR _ssl_SSLSession_ticket_lifetime_hint__doc__ -#else +#if !defined(_ssl_SSLSession_ticket_lifetime_hint_DOCSTR) # define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR NULL #endif #if defined(_SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF) @@ -2360,9 +2292,7 @@ _ssl_SSLSession_ticket_lifetime_hint_get(PySSLSession *self, void *Py_UNUSED(con return return_value; } -#if defined(_ssl_SSLSession_id_HAS_DOCSTR) -# define _ssl_SSLSession_id_DOCSTR _ssl_SSLSession_id__doc__ -#else +#if !defined(_ssl_SSLSession_id_DOCSTR) # define _ssl_SSLSession_id_DOCSTR NULL #endif #if defined(_SSL_SSLSESSION_ID_GETSETDEF) @@ -2387,9 +2317,7 @@ _ssl_SSLSession_id_get(PySSLSession *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_ssl_SSLSession_has_ticket_HAS_DOCSTR) -# define _ssl_SSLSession_has_ticket_DOCSTR _ssl_SSLSession_has_ticket__doc__ -#else +#if !defined(_ssl_SSLSession_has_ticket_DOCSTR) # define _ssl_SSLSession_has_ticket_DOCSTR NULL #endif #if defined(_SSL_SSLSESSION_HAS_TICKET_GETSETDEF) @@ -2839,4 +2767,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=4c2af0c8fab7ec4e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=44ab066d21277ee5 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index fc2d9fe987096d..a57d729bec5733 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -146,7 +146,7 @@ def declare_parser( GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" PyDoc_STRVAR({getset_basename}__doc__, {docstring}); - #define {getset_basename}_HAS_DOCSTR + #define {getset_basename}_DOCSTR {getset_basename}__doc__ """) IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" static {impl_return_type} @@ -157,9 +157,7 @@ def declare_parser( {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, """) GETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" - #if defined({getset_basename}_HAS_DOCSTR) - # define {getset_basename}_DOCSTR {getset_basename}__doc__ - #else + #if !defined({getset_basename}_DOCSTR) # define {getset_basename}_DOCSTR NULL #endif #if defined({getset_name}_GETSETDEF) @@ -170,9 +168,7 @@ def declare_parser( #endif """) SETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" - #if defined({getset_name}_HAS_DOCSTR) - # define {getset_basename}_DOCSTR {getset_basename}__doc__ - #else + #if !defined({getset_basename}_DOCSTR) # define {getset_basename}_DOCSTR NULL #endif #if defined({getset_name}_GETSETDEF) From bfabf96b50b7d6a9c15b298a86ba3633b05a1fd7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Nov 2024 11:39:54 +0100 Subject: [PATCH 047/427] gh-127258: Fix asyncio test_staggered_race_with_eager_tasks() (#127358) Replace the sleep(2) with a task which is blocked forever. --- Lib/test/test_asyncio/test_eager_task_factory.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 31d2a00dbb8c9c..0e2b189f761521 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -220,10 +220,14 @@ async def fail(): await asyncio.sleep(0) raise ValueError("no good") + async def blocked(): + fut = asyncio.Future() + await fut + async def run(): winner, index, excs = await asyncio.staggered.staggered_race( [ - lambda: asyncio.sleep(2, result="sleep2"), + lambda: blocked(), lambda: asyncio.sleep(1, result="sleep1"), lambda: fail() ], From 322b486010095f89271fba51b4b23c35d87c0595 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 29 Nov 2024 19:48:02 +0900 Subject: [PATCH 048/427] gh-126024: optimize UTF-8 decoder for short non-ASCII string (#126025) --- ...-10-27-04-47-28.gh-issue-126024.XCQSqT.rst | 2 + Objects/unicodeobject.c | 304 +++++++++++++++--- 2 files changed, 261 insertions(+), 45 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst new file mode 100644 index 00000000000000..b41fff30433c34 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst @@ -0,0 +1,2 @@ +Optimize decoding of short UTF-8 sequences containing non-ASCII characters +by approximately 15%. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 562e3312b63e9a..ab4f07ed054385 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4979,39 +4979,228 @@ PyUnicode_DecodeUTF8(const char *s, #include "stringlib/codecs.h" #include "stringlib/undef.h" +#if (SIZEOF_SIZE_T == 8) /* Mask to quickly check whether a C 'size_t' contains a non-ASCII, UTF8-encoded char. */ -#if (SIZEOF_SIZE_T == 8) # define ASCII_CHAR_MASK 0x8080808080808080ULL +// used to count codepoints in UTF-8 string. +# define VECTOR_0101 0x0101010101010101ULL +# define VECTOR_00FF 0x00ff00ff00ff00ffULL #elif (SIZEOF_SIZE_T == 4) # define ASCII_CHAR_MASK 0x80808080U +# define VECTOR_0101 0x01010101U +# define VECTOR_00FF 0x00ff00ffU #else # error C 'size_t' size should be either 4 or 8! #endif +#if (defined(__clang__) || defined(__GNUC__)) +#define HAVE_CTZ 1 +static inline unsigned int +ctz(size_t v) +{ + return __builtin_ctzll((unsigned long long)v); +} +#elif defined(_MSC_VER) +#define HAVE_CTZ 1 +static inline unsigned int +ctz(size_t v) +{ + unsigned long pos; +#if SIZEOF_SIZE_T == 4 + _BitScanForward(&pos, v); +#else + _BitScanForward64(&pos, v); +#endif /* SIZEOF_SIZE_T */ + return pos; +} +#endif + +#if HAVE_CTZ +// load p[0]..p[size-1] as a little-endian size_t +// without unaligned access nor read ahead. +static size_t +load_unaligned(const unsigned char *p, size_t size) +{ + assert(size <= SIZEOF_SIZE_T); + union { + size_t s; + unsigned char b[SIZEOF_SIZE_T]; + } u; + u.s = 0; + switch (size) { + case 8: + u.b[7] = p[7]; + _Py_FALLTHROUGH; + case 7: + u.b[6] = p[6]; + _Py_FALLTHROUGH; + case 6: + u.b[5] = p[5]; + _Py_FALLTHROUGH; + case 5: + u.b[4] = p[4]; + _Py_FALLTHROUGH; + case 4: + u.b[3] = p[3]; + _Py_FALLTHROUGH; + case 3: + u.b[2] = p[2]; + _Py_FALLTHROUGH; + case 2: + u.b[1] = p[1]; + _Py_FALLTHROUGH; + case 1: + u.b[0] = p[0]; + break; + case 0: + break; + default: + Py_UNREACHABLE(); + } + return u.s; +} +#endif + +/* + * Find the first non-ASCII character in a byte sequence. + * + * This function scans a range of bytes from `start` to `end` and returns the + * index of the first byte that is not an ASCII character (i.e., has the most + * significant bit set). If all characters in the range are ASCII, it returns + * `end - start`. + */ static Py_ssize_t -ascii_decode(const char *start, const char *end, Py_UCS1 *dest) +find_first_nonascii(const unsigned char *start, const unsigned char *end) { - const char *p = start; + const unsigned char *p = start; + if (end - start >= SIZEOF_SIZE_T) { + const unsigned char *p2 = _Py_ALIGN_UP(p, SIZEOF_SIZE_T); + if (p < p2) { +#if HAVE_CTZ +#if defined(_M_AMD64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) + // x86 and amd64 are little endian and can load unaligned memory. + size_t u = *(const size_t*)p & ASCII_CHAR_MASK; +#else + size_t u = load_unaligned(p, p2 - p) & ASCII_CHAR_MASK; +#endif + if (u) { + return p - start + (ctz(u) - 7) / 8; + } + p = p2; + } +#else + while (p < p2) { + if (*p & 0x80) { + return p - start; + } + p++; + } +#endif + const unsigned char *e = end - SIZEOF_SIZE_T; + while (p <= e) { + size_t u = (*(const size_t *)p) & ASCII_CHAR_MASK; + if (u) { +#if PY_LITTLE_ENDIAN && HAVE_CTZ + return p - start + (ctz(u) - 7) / 8; +#else + // big endian and minor compilers are difficult to test. + // fallback to per byte check. + break; +#endif + } + p += SIZEOF_SIZE_T; + } + } +#if HAVE_CTZ + // we can not use *(const size_t*)p to avoid buffer overrun. + size_t u = load_unaligned(p, end - p) & ASCII_CHAR_MASK; + if (u) { + return p - start + (ctz(u) - 7) / 8; + } + return end - start; +#else + while (p < end) { + if (*p & 0x80) { + break; + } + p++; + } + return p - start; +#endif +} + +static inline int +scalar_utf8_start_char(unsigned int ch) +{ + // 0xxxxxxx or 11xxxxxx are first byte. + return (~ch >> 7 | ch >> 6) & 1; +} + +static inline size_t +vector_utf8_start_chars(size_t v) +{ + return ((~v >> 7) | (v >> 6)) & VECTOR_0101; +} + + +// Count the number of UTF-8 code points in a given byte sequence. +static Py_ssize_t +utf8_count_codepoints(const unsigned char *s, const unsigned char *end) +{ + Py_ssize_t len = 0; + + if (end - s >= SIZEOF_SIZE_T) { + while (!_Py_IS_ALIGNED(s, ALIGNOF_SIZE_T)) { + len += scalar_utf8_start_char(*s++); + } + + while (s + SIZEOF_SIZE_T <= end) { + const unsigned char *e = end; + if (e - s > SIZEOF_SIZE_T * 255) { + e = s + SIZEOF_SIZE_T * 255; + } + Py_ssize_t vstart = 0; + while (s + SIZEOF_SIZE_T <= e) { + size_t v = *(size_t*)s; + size_t vs = vector_utf8_start_chars(v); + vstart += vs; + s += SIZEOF_SIZE_T; + } + vstart = (vstart & VECTOR_00FF) + ((vstart >> 8) & VECTOR_00FF); + vstart += vstart >> 16; +#if SIZEOF_SIZE_T == 8 + vstart += vstart >> 32; +#endif + len += vstart & 0x7ff; + } + } + while (s < end) { + len += scalar_utf8_start_char(*s++); + } + return len; +} + +static Py_ssize_t +ascii_decode(const char *start, const char *end, Py_UCS1 *dest) +{ #if SIZEOF_SIZE_T <= SIZEOF_VOID_P - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T) + if (_Py_IS_ALIGNED(start, ALIGNOF_SIZE_T) && _Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)) { /* Fast path, see in STRINGLIB(utf8_decode) for an explanation. */ - /* Help allocation */ - const char *_p = p; - Py_UCS1 * q = dest; - while (_p + SIZEOF_SIZE_T <= end) { - size_t value = *(const size_t *) _p; + const char *p = start; + Py_UCS1 *q = dest; + while (p + SIZEOF_SIZE_T <= end) { + size_t value = *(const size_t *) p; if (value & ASCII_CHAR_MASK) break; *((size_t *)q) = value; - _p += SIZEOF_SIZE_T; + p += SIZEOF_SIZE_T; q += SIZEOF_SIZE_T; } - p = _p; while (p < end) { if ((unsigned char)*p & 0x80) break; @@ -5020,31 +5209,12 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) return p - start; } #endif - while (p < end) { - /* Fast path, see in STRINGLIB(utf8_decode) in stringlib/codecs.h - for an explanation. */ - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { - /* Help allocation */ - const char *_p = p; - while (_p + SIZEOF_SIZE_T <= end) { - size_t value = *(const size_t *) _p; - if (value & ASCII_CHAR_MASK) - break; - _p += SIZEOF_SIZE_T; - } - p = _p; - if (_p == end) - break; - } - if ((unsigned char)*p & 0x80) - break; - ++p; - } - memcpy(dest, start, p - start); - return p - start; + Py_ssize_t pos = find_first_nonascii((const unsigned char*)start, + (const unsigned char*)end); + memcpy(dest, start, pos); + return pos; } - static int unicode_decode_utf8_impl(_PyUnicodeWriter *writer, const char *starts, const char *s, const char *end, @@ -5188,27 +5358,69 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, return get_latin1_char((unsigned char)s[0]); } - // fast path: try ASCII string. - const char *starts = s; - const char *end = s + size; - PyObject *u = PyUnicode_New(size, 127); - if (u == NULL) { + // I don't know this check is necessary or not. But there is a test + // case that requires size=PY_SSIZE_T_MAX cause MemoryError. + if (PY_SSIZE_T_MAX - sizeof(PyCompactUnicodeObject) < (size_t)size) { + PyErr_NoMemory(); return NULL; } - Py_ssize_t decoded = ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); - if (decoded == size) { + + const char *starts = s; + const char *end = s + size; + + Py_ssize_t pos = find_first_nonascii((const unsigned char*)starts, (const unsigned char*)end); + if (pos == size) { // fast path: ASCII string. + PyObject *u = PyUnicode_New(size, 127); + if (u == NULL) { + return NULL; + } + memcpy(PyUnicode_1BYTE_DATA(u), s, size); if (consumed) { *consumed = size; } return u; } - s += decoded; - size -= decoded; + + int maxchr = 127; + Py_ssize_t maxsize = size; + + unsigned char ch = (unsigned char)(s[pos]); + // error handler other than strict may remove/replace the invalid byte. + // consumed != NULL allows 1~3 bytes remainings. + // 0x80 <= ch < 0xc2 is invalid start byte that cause UnicodeDecodeError. + // otherwise: check the input and decide the maxchr and maxsize to reduce + // reallocation and copy. + if (error_handler == _Py_ERROR_STRICT && !consumed && ch >= 0xc2) { + // we only calculate the number of codepoints and don't determine the exact maxchr. + // This is because writing fast and portable SIMD code to find maxchr is difficult. + // If reallocation occurs for a larger maxchar, knowing the exact number of codepoints + // means that it is no longer necessary to allocate several times the required amount + // of memory. + maxsize = utf8_count_codepoints((const unsigned char *)s, (const unsigned char *)end); + if (ch < 0xc4) { // latin1 + maxchr = 0xff; + } + else if (ch < 0xf0) { // ucs2 + maxchr = 0xffff; + } + else { // ucs4 + maxchr = 0x10ffff; + } + } + PyObject *u = PyUnicode_New(maxsize, maxchr); + if (!u) { + return NULL; + } // Use _PyUnicodeWriter after fast path is failed. _PyUnicodeWriter writer; _PyUnicodeWriter_InitWithBuffer(&writer, u); - writer.pos = decoded; + if (maxchr <= 255) { + memcpy(PyUnicode_1BYTE_DATA(u), s, pos); + s += pos; + size -= pos; + writer.pos = pos; + } if (unicode_decode_utf8_impl(&writer, starts, s, end, error_handler, errors, @@ -5268,7 +5480,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s, const char *errors, Py_ssize_t *consumed) { - return unicode_decode_utf8(s, size, _Py_ERROR_UNKNOWN, errors, consumed); + return unicode_decode_utf8(s, size, + errors ? _Py_ERROR_UNKNOWN : _Py_ERROR_STRICT, + errors, consumed); } From 3afb639f39e89888194d8e74cc498c8da3a58d8e Mon Sep 17 00:00:00 2001 From: biggus-developerus <74741815+biggus-developerus@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:53:34 +0400 Subject: [PATCH 049/427] fix param type in PyObject_HasAttrWithError (docs) (#127403) --- Doc/c-api/object.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 2ac975ff7d1a59..1ae3c46bea46ea 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -85,7 +85,7 @@ Object Protocol instead of the :func:`repr`. -.. c:function:: int PyObject_HasAttrWithError(PyObject *o, const char *attr_name) +.. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This is equivalent to the Python expression ``hasattr(o, attr_name)``. From b14fdadc6c620875a20b7ccc3c9b069e85d8557a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Nov 2024 16:20:38 +0100 Subject: [PATCH 050/427] gh-127208: Reject null character in _imp.create_dynamic() (#127400) _imp.create_dynamic() now rejects embedded null characters in the path and in the module name. --- Lib/test/test_import/__init__.py | 13 +++++++++++++ Python/import.c | 8 +++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 0e39998ebb3783..c52b7f3e09bea1 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1133,6 +1133,19 @@ def test_script_shadowing_stdlib_sys_path_modification(self): stdout, stderr = popen.communicate() self.assertRegex(stdout, expected_error) + def test_create_dynamic_null(self): + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec: + name = "a\x00b" + origin = "abc" + _imp.create_dynamic(Spec()) + + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec2: + name = "abc" + origin = "a\x00b" + _imp.create_dynamic(Spec2()) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): diff --git a/Python/import.c b/Python/import.c index 09fe95fa1fb647..b3c384c27718ce 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1157,12 +1157,14 @@ del_extensions_cache_value(struct extensions_cache_value *value) static void * hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) { - Py_ssize_t str1_len, str2_len; - const char *str1_data = PyUnicode_AsUTF8AndSize(str1, &str1_len); - const char *str2_data = PyUnicode_AsUTF8AndSize(str2, &str2_len); + const char *str1_data = _PyUnicode_AsUTF8NoNUL(str1); + const char *str2_data = _PyUnicode_AsUTF8NoNUL(str2); if (str1_data == NULL || str2_data == NULL) { return NULL; } + Py_ssize_t str1_len = strlen(str1_data); + Py_ssize_t str2_len = strlen(str2_data); + /* Make sure sep and the NULL byte won't cause an overflow. */ assert(SIZE_MAX - str1_len - str2_len > 2); size_t size = str1_len + 1 + str2_len + 1; From 45c5cba318a19dda3ee6f9fc84781cc7a2fbde80 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 29 Nov 2024 21:44:20 +0530 Subject: [PATCH 051/427] gh-127316: fix incorrect assertion in setting `__class__` in free-threading (#127399) --- Lib/test/test_free_threading/test_type.py | 15 +++++++++++++++ Objects/dictobject.c | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 51463b6bb8c1b4..53f6d778bbecbc 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -124,6 +124,21 @@ def work(): for thread in threads: thread.join() + def test_object_class_change(self): + class Base: + def __init__(self): + self.attr = 123 + class ClassA(Base): + pass + class ClassB(Base): + pass + + obj = ClassA() + # keep reference to __dict__ + d = obj.__dict__ + obj.__class__ = ClassB + + def run_one(self, writer_func, reader_func): writer = Thread(target=writer_func) readers = [] diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 49b213eaa817e2..a13d8084d14d66 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -7300,7 +7300,7 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) // We could be called with an unlocked dict when the caller knows the // values are already detached, so we assert after inline values check. - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(mp); assert(mp->ma_values->embedded == 1); assert(mp->ma_values->valid == 1); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); From 15d6506d175780bb29e5fcde654e3860408aa93e Mon Sep 17 00:00:00 2001 From: Ollanta Cuba Gyllensten Date: Fri, 29 Nov 2024 17:20:40 +0100 Subject: [PATCH 052/427] Link to correct class methods in asyncio primitives docs (#127270) --- Doc/library/asyncio-sync.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 3cf8e2737e85dc..77c2e97da11990 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -259,8 +259,8 @@ Condition Note that a task *may* return from this call spuriously, which is why the caller should always re-check the state - and be prepared to :meth:`wait` again. For this reason, you may - prefer to use :meth:`wait_for` instead. + and be prepared to :meth:`~Condition.wait` again. For this reason, you may + prefer to use :meth:`~Condition.wait_for` instead. .. coroutinemethod:: wait_for(predicate) @@ -268,7 +268,7 @@ Condition The predicate must be a callable which result will be interpreted as a boolean value. The method will repeatedly - :meth:`wait` until the predicate evaluates to *true*. The final value is the + :meth:`~Condition.wait` until the predicate evaluates to *true*. The final value is the return value. @@ -434,7 +434,7 @@ Barrier .. coroutinemethod:: abort() Put the barrier into a broken state. This causes any active or future - calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. + calls to :meth:`~Barrier.wait` to fail with the :class:`BrokenBarrierError`. Use this for example if one of the tasks needs to abort, to avoid infinite waiting tasks. From 38264a060a8178d58046e90e9beb8220e3c22046 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 29 Nov 2024 21:03:39 +0000 Subject: [PATCH 053/427] GH-127381: pathlib ABCs: remove `PathBase.lstat()` (#127382) Remove the `PathBase.lstat()` method, which is a trivial variation of `stat()`. No user-facing changes because the pathlib ABCs are still private. --- Lib/pathlib/_abc.py | 12 ++---------- Lib/pathlib/_local.py | 7 +++++++ Lib/test/test_pathlib/test_pathlib.py | 18 +++++++++++++----- Lib/test/test_pathlib/test_pathlib_abc.py | 18 ++++-------------- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 2c243d470d4eda..697f32985ff652 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -438,14 +438,6 @@ def stat(self, *, follow_symlinks=True): """ raise UnsupportedOperation(self._unsupported_msg('stat()')) - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - - # Convenience functions for querying the stat results def exists(self, *, follow_symlinks=True): @@ -505,7 +497,7 @@ def is_symlink(self): Whether this path is a symbolic link. """ try: - return S_ISLNK(self.lstat().st_mode) + return S_ISLNK(self.stat(follow_symlinks=False).st_mode) except (OSError, ValueError): return False @@ -789,7 +781,7 @@ def raise_error(*args): def lstat(path_str): path = self.with_segments(path_str) path._resolving = True - return path.lstat() + return path.stat(follow_symlinks=False) def readlink(path_str): path = self.with_segments(path_str) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index b27f456d375225..25c1e3f44ea7e8 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -542,6 +542,13 @@ def stat(self, *, follow_symlinks=True): """ return os.stat(self, follow_symlinks=follow_symlinks) + def lstat(self): + """ + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. + """ + return os.lstat(self) + def exists(self, *, follow_symlinks=True): """ Whether this path exists. diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 6a994f890da616..2c48eeeda145d0 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -546,12 +546,9 @@ def tempdir(self): self.addCleanup(os_helper.rmtree, d) return d - def test_matches_pathbase_api(self): - our_names = {name for name in dir(self.cls) if name[0] != '_'} - our_names.remove('is_reserved') # only present in PurePath + def test_matches_pathbase_docstrings(self): path_names = {name for name in dir(pathlib._abc.PathBase) if name[0] != '_'} - self.assertEqual(our_names, path_names) - for attr_name in our_names: + for attr_name in path_names: if attr_name == 'parser': # On Windows, Path.parser is ntpath, but PathBase.parser is # posixpath, and so their docstrings differ. @@ -1357,6 +1354,17 @@ def test_symlink_to_unsupported(self): with self.assertRaises(pathlib.UnsupportedOperation): q.symlink_to(p) + @needs_symlinks + def test_lstat(self): + p = self.cls(self.base)/ 'linkA' + st = p.stat() + self.assertNotEqual(st, p.lstat()) + + def test_lstat_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.lstat()) + def test_is_junction(self): P = self.cls(self.base) diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index aaa30a17f2af14..7ca35e3dc7ee00 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1351,7 +1351,6 @@ def test_unsupported_operation(self): p = self.cls('') e = UnsupportedOperation self.assertRaises(e, p.stat) - self.assertRaises(e, p.lstat) self.assertRaises(e, p.exists) self.assertRaises(e, p.samefile, 'foo') self.assertRaises(e, p.is_dir) @@ -2671,17 +2670,6 @@ def test_stat_no_follow_symlinks_nosymlink(self): st = p.stat() self.assertEqual(st, p.stat(follow_symlinks=False)) - @needs_symlinks - def test_lstat(self): - p = self.cls(self.base)/ 'linkA' - st = p.stat() - self.assertNotEqual(st, p.lstat()) - - def test_lstat_nosymlink(self): - p = self.cls(self.base) / 'fileA' - st = p.stat() - self.assertEqual(st, p.lstat()) - def test_is_dir(self): P = self.cls(self.base) self.assertTrue((P / 'dirA').is_dir()) @@ -2868,11 +2856,13 @@ def test_delete_dir(self): base = self.cls(self.base) base.joinpath('dirA')._delete() self.assertRaises(FileNotFoundError, base.joinpath('dirA').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirA', 'linkC').lstat) + self.assertRaises(FileNotFoundError, base.joinpath('dirA', 'linkC').stat, + follow_symlinks=False) base.joinpath('dirB')._delete() self.assertRaises(FileNotFoundError, base.joinpath('dirB').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'fileB').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'linkD').lstat) + self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'linkD').stat, + follow_symlinks=False) base.joinpath('dirC')._delete() self.assertRaises(FileNotFoundError, base.joinpath('dirC').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD').stat) From 49f15d8667e4755876698a8daa13bab6acee5fa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 30 Nov 2024 01:25:55 +0100 Subject: [PATCH 054/427] gh-119786: improve internal docs on `co_linetable` (#123198) --- InternalDocs/code_objects.md | 87 +++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/InternalDocs/code_objects.md b/InternalDocs/code_objects.md index bee4a9d0a08915..d4e28c6b238b48 100644 --- a/InternalDocs/code_objects.md +++ b/InternalDocs/code_objects.md @@ -1,4 +1,3 @@ - # Code objects A `CodeObject` is a builtin Python type that represents a compiled executable, @@ -43,7 +42,7 @@ so a compact format is very important. Note that traceback objects don't store all this information -- they store the start line number, for backward compatibility, and the "last instruction" value. The rest can be computed from the last instruction (`tb_lasti`) with the help of the -locations table. For Python code, there is a convenience method +locations table. For Python code, there is a convenience method (`codeobject.co_positions`)[https://docs.python.org/dev/reference/datamodel.html#codeobject.co_positions] which returns an iterator of `({line}, {endline}, {column}, {endcolumn})` tuples, one per instruction. @@ -75,9 +74,11 @@ returned by the `co_positions()` iterator. > See [`Objects/lnotab_notes.txt`](../Objects/lnotab_notes.txt) for more details. `co_linetable` consists of a sequence of location entries. -Each entry starts with a byte with the most significant bit set, followed by zero or more bytes with the most significant bit unset. +Each entry starts with a byte with the most significant bit set, followed by +zero or more bytes with the most significant bit unset. Each entry contains the following information: + * The number of code units covered by this entry (length) * The start line * The end line @@ -86,54 +87,88 @@ Each entry contains the following information: The first byte has the following format: -Bit 7 | Bits 3-6 | Bits 0-2 - ---- | ---- | ---- - 1 | Code | Length (in code units) - 1 +| Bit 7 | Bits 3-6 | Bits 0-2 | +|-------|----------|----------------------------| +| 1 | Code | Length (in code units) - 1 | The codes are enumerated in the `_PyCodeLocationInfoKind` enum. -## Variable-length integer encodings +### Variable-length integer encodings -Integers are often encoded using a variable-length integer encoding +Integers are often encoded using a variable length integer encoding -### Unsigned integers (`varint`) +#### Unsigned integers (`varint`) Unsigned integers are encoded in 6-bit chunks, least significant first. Each chunk but the last has bit 6 set. For example: * 63 is encoded as `0x3f` -* 200 is encoded as `0x48`, `0x03` +* 200 is encoded as `0x48`, `0x03` since ``200 = (0x03 << 6) | 0x48``. + +The following helper can be used to convert an integer into a `varint`: + +```py +def encode_varint(s): + ret = [] + while s >= 64: + ret.append(((s & 0x3F) | 0x40) & 0x3F) + s >>= 6 + ret.append(s & 0x3F) + return bytes(ret) +``` + +To convert a `varint` into an unsigned integer: + +```py +def decode_varint(chunks): + ret = 0 + for chunk in reversed(chunks): + ret = (ret << 6) | chunk + return ret +``` -### Signed integers (`svarint`) +#### Signed integers (`svarint`) Signed integers are encoded by converting them to unsigned integers, using the following function: -```Python -def convert(s): + +```py +def svarint_to_varint(s): if s < 0: - return ((-s)<<1) | 1 + return ((-s) << 1) | 1 else: - return (s<<1) + return s << 1 +``` + +To convert a `varint` into a signed integer: + +```py +def varint_to_svarint(uval): + return -(uval >> 1) if uval & 1 else (uval >> 1) ``` -*Location entries* +### Location entries The meaning of the codes and the following bytes are as follows: -Code | Meaning | Start line | End line | Start column | End column - ---- | ---- | ---- | ---- | ---- | ---- - 0-9 | Short form | Δ 0 | Δ 0 | See below | See below - 10-12 | One line form | Δ (code - 10) | Δ 0 | unsigned byte | unsigned byte - 13 | No column info | Δ svarint | Δ 0 | None | None - 14 | Long form | Δ svarint | Δ varint | varint | varint - 15 | No location | None | None | None | None +| Code | Meaning | Start line | End line | Start column | End column | +|-------|----------------|---------------|----------|---------------|---------------| +| 0-9 | Short form | Δ 0 | Δ 0 | See below | See below | +| 10-12 | One line form | Δ (code - 10) | Δ 0 | unsigned byte | unsigned byte | +| 13 | No column info | Δ svarint | Δ 0 | None | None | +| 14 | Long form | Δ svarint | Δ varint | varint | varint | +| 15 | No location | None | None | None | None | The Δ means the value is encoded as a delta from another value: + * Start line: Delta from the previous start line, or `co_firstlineno` for the first entry. -* End line: Delta from the start line +* End line: Delta from the start line. + +### The short forms -*The short forms* +Codes 0-9 are the short forms. The short form consists of two bytes, +the second byte holding additional column information. The code is the +start column divided by 8 (and rounded down). -Codes 0-9 are the short forms. The short form consists of two bytes, the second byte holding additional column information. The code is the start column divided by 8 (and rounded down). * Start column: `(code*8) + ((second_byte>>4)&7)` * End column: `start_column + (second_byte&15)` From 7043bbd1ca6f0d84ad2211dd60114535ba3d51fc Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 30 Nov 2024 21:52:37 +0900 Subject: [PATCH 055/427] gh-127417: fix UTF-8 decoder optimization on AIX (#127433) --- Objects/unicodeobject.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index ab4f07ed054385..33fa21d4c7d1bf 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5014,21 +5014,26 @@ ctz(size_t v) #endif /* SIZEOF_SIZE_T */ return pos; } +#else +#define HAVE_CTZ 0 #endif -#if HAVE_CTZ -// load p[0]..p[size-1] as a little-endian size_t -// without unaligned access nor read ahead. +#if HAVE_CTZ && PY_LITTLE_ENDIAN +// load p[0]..p[size-1] as a size_t without unaligned access nor read ahead. static size_t load_unaligned(const unsigned char *p, size_t size) { - assert(size <= SIZEOF_SIZE_T); union { size_t s; unsigned char b[SIZEOF_SIZE_T]; } u; u.s = 0; + // This switch statement assumes little endian because: + // * union is faster than bitwise or and shift. + // * big endian machine is rare and hard to maintain. switch (size) { + default: +#if SIZEOF_SIZE_T == 8 case 8: u.b[7] = p[7]; _Py_FALLTHROUGH; @@ -5041,6 +5046,7 @@ load_unaligned(const unsigned char *p, size_t size) case 5: u.b[4] = p[4]; _Py_FALLTHROUGH; +#endif case 4: u.b[3] = p[3]; _Py_FALLTHROUGH; @@ -5055,8 +5061,6 @@ load_unaligned(const unsigned char *p, size_t size) break; case 0: break; - default: - Py_UNREACHABLE(); } return u.s; } @@ -5077,8 +5081,8 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) if (end - start >= SIZEOF_SIZE_T) { const unsigned char *p2 = _Py_ALIGN_UP(p, SIZEOF_SIZE_T); +#if PY_LITTLE_ENDIAN && HAVE_CTZ if (p < p2) { -#if HAVE_CTZ #if defined(_M_AMD64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) // x86 and amd64 are little endian and can load unaligned memory. size_t u = *(const size_t*)p & ASCII_CHAR_MASK; @@ -5086,11 +5090,11 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) size_t u = load_unaligned(p, p2 - p) & ASCII_CHAR_MASK; #endif if (u) { - return p - start + (ctz(u) - 7) / 8; + return (ctz(u) - 7) / 8; } p = p2; } -#else +#else /* PY_LITTLE_ENDIAN && HAVE_CTZ */ while (p < p2) { if (*p & 0x80) { return p - start; @@ -5113,7 +5117,7 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) p += SIZEOF_SIZE_T; } } -#if HAVE_CTZ +#if PY_LITTLE_ENDIAN && HAVE_CTZ // we can not use *(const size_t*)p to avoid buffer overrun. size_t u = load_unaligned(p, end - p) & ASCII_CHAR_MASK; if (u) { From 33ce8dcf791721fea563715f681dc1593a35b83b Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Sun, 1 Dec 2024 00:01:15 +0900 Subject: [PATCH 056/427] Docs: Fix incorrect indents in `c-api/type.rst` (#127449) --- Doc/c-api/type.rst | 48 ++++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 86d3967d9fb577..444b3456f051d8 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -529,19 +529,19 @@ The following functions and structs are used to create The following “offset” fields cannot be set using :c:type:`PyType_Slot`: - * :c:member:`~PyTypeObject.tp_weaklistoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) - * :c:member:`~PyTypeObject.tp_dictoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) - * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (use ``"__vectorcalloffset__"`` in - :ref:`PyMemberDef `) - - If it is not possible to switch to a ``MANAGED`` flag (for example, - for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. - See :ref:`PyMemberDef documentation ` - for details. + * :c:member:`~PyTypeObject.tp_weaklistoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) + * :c:member:`~PyTypeObject.tp_dictoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) + * :c:member:`~PyTypeObject.tp_vectorcall_offset` + (use ``"__vectorcalloffset__"`` in + :ref:`PyMemberDef `) + + If it is not possible to switch to a ``MANAGED`` flag (for example, + for vectorcall or to support Python older than 3.12), specify the + offset in :c:member:`Py_tp_members `. + See :ref:`PyMemberDef documentation ` + for details. The following internal fields cannot be set at all when creating a heap type: @@ -557,20 +557,18 @@ The following functions and structs are used to create To avoid issues, use the *bases* argument of :c:func:`PyType_FromSpecWithBases` instead. - .. versionchanged:: 3.9 - - Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + .. versionchanged:: 3.9 + Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. - .. versionchanged:: 3.11 - :c:member:`~PyBufferProcs.bf_getbuffer` and - :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the :ref:`limited API `. + .. versionchanged:: 3.11 + :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` are now available + under the :ref:`limited API `. - .. versionchanged:: 3.14 - - The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation - for details. + .. versionchanged:: 3.14 + The field :c:member:`~PyTypeObject.tp_vectorcall` can now set + using ``Py_tp_vectorcall``. See the field's documentation + for details. .. c:member:: void *pfunc From 4e0a4cafe8d8ecb43db62aed1d5671af583104e7 Mon Sep 17 00:00:00 2001 From: alm Date: Sat, 30 Nov 2024 18:07:54 +0200 Subject: [PATCH 057/427] summarize: Fix typo in stats (#127450) --- Tools/scripts/summarize_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 5793e5c649d6b3..abfdea78253760 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -483,7 +483,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: ): (trace_too_long, attempts), Doc( "Trace too short", - "A potential trace is abandoced because it it too short.", + "A potential trace is abandoned because it it too short.", ): (trace_too_short, attempts), Doc( "Inner loop found", "A trace is truncated because it has an inner loop" From 328187cc4fcdd578db42cf6a16c197c3382157a7 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 30 Nov 2024 18:39:39 +0000 Subject: [PATCH 058/427] GH-127381: pathlib ABCs: remove `PathBase.cwd()` and `home()` (#127427) These classmethods presume that the user has retained the original `__init__()` signature, which may not be the case. Also, many virtual filesystems don't provide current or home directories. --- Lib/pathlib/_abc.py | 15 --------------- Lib/pathlib/_local.py | 17 +++++++++++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 2 -- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 697f32985ff652..2b314b6c9a16bf 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -735,27 +735,12 @@ def absolute(self): # Treat the root directory as the current working directory. return self.with_segments('/', *self._raw_paths) - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls().absolute() - def expanduser(self): """ Return a new path with expanded ~ and ~user constructs (as returned by os.path.expanduser) """ raise UnsupportedOperation(self._unsupported_msg('expanduser()')) - @classmethod - def home(cls): - """Return a new path pointing to expanduser('~'). - """ - return cls("~").expanduser() - def readlink(self): """ Return the path to which the symbolic link points. diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 25c1e3f44ea7e8..b5d9dc49f58463 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -726,6 +726,14 @@ def absolute(self): tail.extend(self._tail) return self._from_parsed_parts(drive, root, tail) + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + cwd = os.getcwd() + path = cls(cwd) + path._str = cwd # getcwd() returns a normalized path + return path + def resolve(self, strict=False): """ Make the path absolute, resolving all symlinks on the way and also @@ -907,6 +915,15 @@ def expanduser(self): return self + @classmethod + def home(cls): + """Return a new path pointing to expanduser('~'). + """ + homedir = os.path.expanduser("~") + if homedir == "~": + raise RuntimeError("Could not determine home directory.") + return cls(homedir) + @classmethod def from_uri(cls, uri): """Return a new path from the given 'file' URI.""" diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 7ca35e3dc7ee00..af94ac039808f0 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1371,9 +1371,7 @@ def test_unsupported_operation(self): self.assertRaises(e, p.rglob, '*') self.assertRaises(e, lambda: list(p.walk())) self.assertRaises(e, p.absolute) - self.assertRaises(e, P.cwd) self.assertRaises(e, p.expanduser) - self.assertRaises(e, p.home) self.assertRaises(e, p.readlink) self.assertRaises(e, p.symlink_to, 'foo') self.assertRaises(e, p.hardlink_to, 'foo') From 46bfd26fb294a8769ef6d0c056ee6c9df022a037 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 1 Dec 2024 01:33:23 -0500 Subject: [PATCH 059/427] gh-127165: Disallow embedded NULL characters in `_interpreters` (#127199) --- Lib/test/test_interpreters/test_api.py | 4 ++++ Python/crossinterp.c | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index a9befbba64daa0..01856d9bf67657 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1649,6 +1649,10 @@ def test_set___main___attrs(self): self.assertIs(after2, None) self.assertEqual(after3.type.__name__, 'AssertionError') + with self.assertRaises(ValueError): + # GH-127165: Embedded NULL characters broke the lookup + _interpreters.set___main___attrs(interpid, {"\x00": 1}) + with self.subTest('from C-API'): with self.interpreter_from_capi() as interpid: with self.assertRaisesRegex(InterpreterError, 'unrecognized'): diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 7aaa045f375cf0..0a106ad636bfe8 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -342,6 +342,11 @@ _copy_string_obj_raw(PyObject *strobj, Py_ssize_t *p_size) return NULL; } + if (size != (Py_ssize_t)strlen(str)) { + PyErr_SetString(PyExc_ValueError, "found embedded NULL character"); + return NULL; + } + char *copied = PyMem_RawMalloc(size+1); if (copied == NULL) { PyErr_NoMemory(); From 11c01092d5fa8f02c867a7f1f3c135ce63db4838 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 30 Nov 2024 22:44:43 -0800 Subject: [PATCH 060/427] Add the missing `f` on an f-string error message in multiprocessing. (GH-127462) --- Lib/multiprocessing/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 996887cb713942..710aba9685efda 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -963,7 +963,7 @@ def answer_challenge(connection, authkey: bytes): f'Protocol error, expected challenge: {message=}') message = message[len(_CHALLENGE):] if len(message) < _MD5ONLY_MESSAGE_LENGTH: - raise AuthenticationError('challenge too short: {len(message)} bytes') + raise AuthenticationError(f'challenge too short: {len(message)} bytes') digest = _create_response(authkey, message) connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message From a880358af03d9cab37f7db04385c5a97051b03b6 Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Sun, 1 Dec 2024 06:15:44 -0300 Subject: [PATCH 061/427] gh-127356: Fix prepend doctrees directory for gettext target (#127357) --- Doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/Makefile b/Doc/Makefile index 22e43ee3e542ee..4a704ad58b33d3 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -144,7 +144,7 @@ pydoc-topics: build .PHONY: gettext gettext: BUILDER = gettext -gettext: SPHINXOPTS += -d build/doctrees-gettext +gettext: override SPHINXOPTS := -d build/doctrees-gettext $(SPHINXOPTS) gettext: build .PHONY: htmlview From 1bc4f076d193ad157bdc69a1d62685a15f95113f Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 1 Dec 2024 08:57:03 -0800 Subject: [PATCH 062/427] gh-127321: Avoid stopping at an opcode without an associated line number for breakpoint() (#127457) --- Lib/pdb.py | 7 +++++++ Lib/test/test_pdb.py | 16 ++++++++++++++++ ...024-11-30-21-46-15.gh-issue-127321.M78fBv.rst | 1 + 3 files changed, 24 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index b7f6fd4323407e..10d1923cdad2d6 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -438,6 +438,13 @@ def user_line(self, frame): if (self.mainpyfile != self.canonic(frame.f_code.co_filename)): return self._wait_for_mainpyfile = False + if self.trace_opcodes: + # GH-127321 + # We want to avoid stopping at an opcode that does not have + # an associated line number because pdb does not like it + if frame.f_lineno is None: + self.set_stepinstr() + return self.bp_commands(frame) self.interaction(frame, None) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index e5f9848319021a..48a4c568651879 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2931,6 +2931,22 @@ def test_pdb_issue_gh_108976(): (Pdb) continue """ +def test_pdb_issue_gh_127321(): + """See GH-127321 + breakpoint() should stop at a opcode that has a line number + >>> def test_function(): + ... import pdb; pdb_instance = pdb.Pdb(nosigint=True, readrc=False) + ... [1, 2] and pdb_instance.set_trace() + ... a = 1 + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'continue' + ... ]): + ... test_function() + > (4)test_function() + -> a = 1 + (Pdb) continue + """ + def test_pdb_issue_gh_80731(): """See GH-80731 diff --git a/Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst b/Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst new file mode 100644 index 00000000000000..69b6ce68a47509 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst @@ -0,0 +1 @@ +:func:`pdb.set_trace` will not stop at an opcode that does not have an associated line number anymore. From 04673d2f14414fce7a2372de3048190f66488e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 1 Dec 2024 18:12:22 +0100 Subject: [PATCH 063/427] gh-119786: cleanup internal docs and fix internal links (#127485) --- InternalDocs/README.md | 1 - InternalDocs/adaptive.md | 8 ++- InternalDocs/changing_grammar.md | 4 +- InternalDocs/compiler.md | 112 ++++++++++++++--------------- InternalDocs/exception_handling.md | 65 +++++++++-------- InternalDocs/frames.md | 1 + InternalDocs/garbage_collector.md | 3 +- InternalDocs/generators.md | 1 - InternalDocs/interpreter.md | 1 - InternalDocs/parser.md | 101 +++++++++++++------------- InternalDocs/string_interning.md | 3 + 11 files changed, 152 insertions(+), 148 deletions(-) diff --git a/InternalDocs/README.md b/InternalDocs/README.md index f6aa3db3b384af..8cdd06d189f362 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -1,4 +1,3 @@ - # CPython Internals Documentation The documentation in this folder is intended for CPython maintainers. diff --git a/InternalDocs/adaptive.md b/InternalDocs/adaptive.md index 4ae9e85b387f39..7cfa8e52310460 100644 --- a/InternalDocs/adaptive.md +++ b/InternalDocs/adaptive.md @@ -96,6 +96,7 @@ quality of specialization and keeping the overhead of specialization low. Specialized instructions must be fast. In order to be fast, specialized instructions should be tailored for a particular set of values that allows them to: + 1. Verify that incoming value is part of that set with low overhead. 2. Perform the operation quickly. @@ -107,9 +108,11 @@ For example, `LOAD_GLOBAL_MODULE` is specialized for `globals()` dictionaries that have a keys with the expected version. This can be tested quickly: + * `globals->keys->dk_version == expected_version` and the operation can be performed quickly: + * `value = entries[cache->index].me_value;`. Because it is impossible to measure the performance of an instruction without @@ -122,10 +125,11 @@ base instruction. ### Implementation of specialized instructions In general, specialized instructions should be implemented in two parts: + 1. A sequence of guards, each of the form - `DEOPT_IF(guard-condition-is-false, BASE_NAME)`. + `DEOPT_IF(guard-condition-is-false, BASE_NAME)`. 2. The operation, which should ideally have no branches and - a minimum number of dependent memory accesses. + a minimum number of dependent memory accesses. In practice, the parts may overlap, as data required for guards can be re-used in the operation. diff --git a/InternalDocs/changing_grammar.md b/InternalDocs/changing_grammar.md index 1a5eebdc1418dc..c6b895135a360d 100644 --- a/InternalDocs/changing_grammar.md +++ b/InternalDocs/changing_grammar.md @@ -32,7 +32,7 @@ Below is a checklist of things that may need to change. [`Include/internal/pycore_ast.h`](../Include/internal/pycore_ast.h) and [`Python/Python-ast.c`](../Python/Python-ast.c). -* [`Parser/lexer/`](../Parser/lexer/) contains the tokenization code. +* [`Parser/lexer/`](../Parser/lexer) contains the tokenization code. This is where you would add a new type of comment or string literal, for example. * [`Python/ast.c`](../Python/ast.c) will need changes to validate AST objects @@ -60,4 +60,4 @@ Below is a checklist of things that may need to change. to the tokenizer. * Documentation must be written! Specifically, one or more of the pages in - [`Doc/reference/`](../Doc/reference/) will need to be updated. + [`Doc/reference/`](../Doc/reference) will need to be updated. diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index ed4cfb23ca51f7..9e99f348acbd8f 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -1,4 +1,3 @@ - Compiler design =============== @@ -7,8 +6,8 @@ Abstract In CPython, the compilation from source code to bytecode involves several steps: -1. Tokenize the source code [Parser/lexer/](../Parser/lexer/) - and [Parser/tokenizer/](../Parser/tokenizer/). +1. Tokenize the source code [Parser/lexer/](../Parser/lexer) + and [Parser/tokenizer/](../Parser/tokenizer). 2. Parse the stream of tokens into an Abstract Syntax Tree [Parser/parser.c](../Parser/parser.c). 3. Transform AST into an instruction sequence @@ -134,9 +133,8 @@ this case) a `stmt_ty` struct with the appropriate initialization. The `FunctionDef()` constructor function sets 'kind' to `FunctionDef_kind` and initializes the *name*, *args*, *body*, and *attributes* fields. -See also -[Green Tree Snakes - The missing Python AST docs](https://greentreesnakes.readthedocs.io/en/latest) - by Thomas Kluyver. +See also [Green Tree Snakes - The missing Python AST docs]( +https://greentreesnakes.readthedocs.io/en/latest) by Thomas Kluyver. Memory management ================= @@ -260,12 +258,12 @@ manually -- `generic`, `identifier` and `int`. These types are found in [Include/internal/pycore_asdl.h](../Include/internal/pycore_asdl.h). Functions and macros for creating `asdl_xx_seq *` types are as follows: -`_Py_asdl_generic_seq_new(Py_ssize_t, PyArena *)` - Allocate memory for an `asdl_generic_seq` of the specified length -`_Py_asdl_identifier_seq_new(Py_ssize_t, PyArena *)` - Allocate memory for an `asdl_identifier_seq` of the specified length -`_Py_asdl_int_seq_new(Py_ssize_t, PyArena *)` - Allocate memory for an `asdl_int_seq` of the specified length +* `_Py_asdl_generic_seq_new(Py_ssize_t, PyArena *)`: + Allocate memory for an `asdl_generic_seq` of the specified length +* `_Py_asdl_identifier_seq_new(Py_ssize_t, PyArena *)`: + Allocate memory for an `asdl_identifier_seq` of the specified length +* `_Py_asdl_int_seq_new(Py_ssize_t, PyArena *)`: + Allocate memory for an `asdl_int_seq` of the specified length In addition to the three types mentioned above, some ASDL sequence types are automatically generated by [Parser/asdl_c.py](../Parser/asdl_c.py) and found in @@ -273,20 +271,20 @@ automatically generated by [Parser/asdl_c.py](../Parser/asdl_c.py) and found in Macros for using both manually defined and automatically generated ASDL sequence types are as follows: -`asdl_seq_GET(asdl_xx_seq *, int)` - Get item held at a specific position in an `asdl_xx_seq` -`asdl_seq_SET(asdl_xx_seq *, int, stmt_ty)` - Set a specific index in an `asdl_xx_seq` to the specified value +* `asdl_seq_GET(asdl_xx_seq *, int)`: + Get item held at a specific position in an `asdl_xx_seq` +* `asdl_seq_SET(asdl_xx_seq *, int, stmt_ty)`: + Set a specific index in an `asdl_xx_seq` to the specified value -Untyped counterparts exist for some of the typed macros. These are useful +Untyped counterparts exist for some of the typed macros. These are useful when a function needs to manipulate a generic ASDL sequence: -`asdl_seq_GET_UNTYPED(asdl_seq *, int)` - Get item held at a specific position in an `asdl_seq` -`asdl_seq_SET_UNTYPED(asdl_seq *, int, stmt_ty)` - Set a specific index in an `asdl_seq` to the specified value -`asdl_seq_LEN(asdl_seq *)` - Return the length of an `asdl_seq` or `asdl_xx_seq` +* `asdl_seq_GET_UNTYPED(asdl_seq *, int)`: + Get item held at a specific position in an `asdl_seq` +* `asdl_seq_SET_UNTYPED(asdl_seq *, int, stmt_ty)`: + Set a specific index in an `asdl_seq` to the specified value +* `asdl_seq_LEN(asdl_seq *)`: + Return the length of an `asdl_seq` or `asdl_xx_seq` Note that typed macros and functions are recommended over their untyped counterparts. Typed macros carry out checks in debug mode and aid @@ -379,33 +377,33 @@ arguments to a node that used the '*' modifier). Emission of bytecode is handled by the following macros: -* `ADDOP(struct compiler *, location, int)` - add a specified opcode -* `ADDOP_IN_SCOPE(struct compiler *, location, int)` - like `ADDOP`, but also exits current scope; used for adding return value - opcodes in lambdas and closures -* `ADDOP_I(struct compiler *, location, int, Py_ssize_t)` - add an opcode that takes an integer argument -* `ADDOP_O(struct compiler *, location, int, PyObject *, TYPE)` - add an opcode with the proper argument based on the position of the - specified PyObject in PyObject sequence object, but with no handling of - mangled names; used for when you - need to do named lookups of objects such as globals, consts, or - parameters where name mangling is not possible and the scope of the - name is known; *TYPE* is the name of PyObject sequence - (`names` or `varnames`) -* `ADDOP_N(struct compiler *, location, int, PyObject *, TYPE)` - just like `ADDOP_O`, but steals a reference to PyObject -* `ADDOP_NAME(struct compiler *, location, int, PyObject *, TYPE)` - just like `ADDOP_O`, but name mangling is also handled; used for - attribute loading or importing based on name -* `ADDOP_LOAD_CONST(struct compiler *, location, PyObject *)` - add the `LOAD_CONST` opcode with the proper argument based on the - position of the specified PyObject in the consts table. -* `ADDOP_LOAD_CONST_NEW(struct compiler *, location, PyObject *)` - just like `ADDOP_LOAD_CONST_NEW`, but steals a reference to PyObject -* `ADDOP_JUMP(struct compiler *, location, int, basicblock *)` - create a jump to a basic block +* `ADDOP(struct compiler *, location, int)`: + add a specified opcode +* `ADDOP_IN_SCOPE(struct compiler *, location, int)`: + like `ADDOP`, but also exits current scope; used for adding return value + opcodes in lambdas and closures +* `ADDOP_I(struct compiler *, location, int, Py_ssize_t)`: + add an opcode that takes an integer argument +* `ADDOP_O(struct compiler *, location, int, PyObject *, TYPE)`: + add an opcode with the proper argument based on the position of the + specified PyObject in PyObject sequence object, but with no handling of + mangled names; used for when you + need to do named lookups of objects such as globals, consts, or + parameters where name mangling is not possible and the scope of the + name is known; *TYPE* is the name of PyObject sequence + (`names` or `varnames`) +* `ADDOP_N(struct compiler *, location, int, PyObject *, TYPE)`: + just like `ADDOP_O`, but steals a reference to PyObject +* `ADDOP_NAME(struct compiler *, location, int, PyObject *, TYPE)`: + just like `ADDOP_O`, but name mangling is also handled; used for + attribute loading or importing based on name +* `ADDOP_LOAD_CONST(struct compiler *, location, PyObject *)`: + add the `LOAD_CONST` opcode with the proper argument based on the + position of the specified PyObject in the consts table. +* `ADDOP_LOAD_CONST_NEW(struct compiler *, location, PyObject *)`: + just like `ADDOP_LOAD_CONST_NEW`, but steals a reference to PyObject +* `ADDOP_JUMP(struct compiler *, location, int, basicblock *)`: + create a jump to a basic block The `location` argument is a struct with the source location to be associated with this instruction. It is typically extracted from an @@ -433,7 +431,7 @@ Finally, the sequence of pseudo-instructions is converted into actual bytecode. This includes transforming pseudo instructions into actual instructions, converting jump targets from logical labels to relative offsets, and construction of the [exception table](exception_handling.md) and -[locations table](locations.md). +[locations table](code_objects.md#source-code-locations). The bytecode and tables are then wrapped into a `PyCodeObject` along with additional metadata, including the `consts` and `names` arrays, information about function reference to the source code (filename, etc). All of this is implemented by @@ -453,7 +451,7 @@ in [Python/ceval.c](../Python/ceval.c). Important files =============== -* [Parser/](../Parser/) +* [Parser/](../Parser) * [Parser/Python.asdl](../Parser/Python.asdl): ASDL syntax file. @@ -534,7 +532,7 @@ Important files * [Python/instruction_sequence.c](../Python/instruction_sequence.c): A data structure representing a sequence of bytecode-like pseudo-instructions. -* [Include/](../Include/) +* [Include/](../Include) * [Include/cpython/code.h](../Include/cpython/code.h) : Header file for [Objects/codeobject.c](../Objects/codeobject.c); @@ -556,7 +554,7 @@ Important files : Declares `_PyAST_Validate()` external (from [Python/ast.c](../Python/ast.c)). * [Include/internal/pycore_symtable.h](../Include/internal/pycore_symtable.h) - : Header for [Python/symtable.c](../Python/symtable.c). + : Header for [Python/symtable.c](../Python/symtable.c). `struct symtable` and `PySTEntryObject` are defined here. * [Include/internal/pycore_parser.h](../Include/internal/pycore_parser.h) @@ -570,7 +568,7 @@ Important files by [Tools/cases_generator/opcode_id_generator.py](../Tools/cases_generator/opcode_id_generator.py). -* [Objects/](../Objects/) +* [Objects/](../Objects) * [Objects/codeobject.c](../Objects/codeobject.c) : Contains PyCodeObject-related code. @@ -579,7 +577,7 @@ Important files : Contains the `frame_setlineno()` function which should determine whether it is allowed to make a jump between two points in a bytecode. -* [Lib/](../Lib/) +* [Lib/](../Lib) * [Lib/opcode.py](../Lib/opcode.py) : opcode utilities exposed to Python. @@ -591,7 +589,7 @@ Important files Objects ======= -* [Locations](locations.md): Describes the location table +* [Locations](code_objects.md#source-code-locations): Describes the location table * [Frames](frames.md): Describes frames and the frame stack * [Objects/object_layout.md](../Objects/object_layout.md): Describes object layout for 3.11 and later * [Exception Handling](exception_handling.md): Describes the exception table diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md index 14066a5864b4da..28589787e1fad7 100644 --- a/InternalDocs/exception_handling.md +++ b/InternalDocs/exception_handling.md @@ -87,10 +87,10 @@ offset of the raising instruction should be pushed to the stack. Handling an exception, once an exception table entry is found, consists of the following steps: - 1. pop values from the stack until it matches the stack depth for the handler. - 2. if `lasti` is true, then push the offset that the exception was raised at. - 3. push the exception to the stack. - 4. jump to the target offset and resume execution. +1. pop values from the stack until it matches the stack depth for the handler. +2. if `lasti` is true, then push the offset that the exception was raised at. +3. push the exception to the stack. +4. jump to the target offset and resume execution. Reraising Exceptions and `lasti` @@ -107,13 +107,12 @@ Format of the exception table ----------------------------- Conceptually, the exception table consists of a sequence of 5-tuples: -``` - 1. `start-offset` (inclusive) - 2. `end-offset` (exclusive) - 3. `target` - 4. `stack-depth` - 5. `push-lasti` (boolean) -``` + +1. `start-offset` (inclusive) +2. `end-offset` (exclusive) +3. `target` +4. `stack-depth` +5. `push-lasti` (boolean) All offsets and lengths are in code units, not bytes. @@ -123,18 +122,19 @@ For it to be searchable quickly, we need to support binary search giving us log( Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: - `start, size, target, depth, push-lasti`. +`start, size, target, depth, push-lasti`. Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each code unit takes 2 bytes. It also happens that depth is generally quite small. So, we need to encode: + ``` - `start` (up to 30 bits) - `size` (up to 30 bits) - `target` (up to 30 bits) - `depth` (up to ~8 bits) - `lasti` (1 bit) +start (up to 30 bits) +size (up to 30 bits) +target (up to 30 bits) +depth (up to ~8 bits) +lasti (1 bit) ``` We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. @@ -145,29 +145,32 @@ The 8 bits of a byte are (msb left) SXdddddd where S is the start bit. X is the In addition, we combine `depth` and `lasti` into a single value, `((depth<<1)+lasti)`, before encoding. For example, the exception entry: + ``` - `start`: 20 - `end`: 28 - `target`: 100 - `depth`: 3 - `lasti`: False +start: 20 +end: 28 +target: 100 +depth: 3 +lasti: False ``` is encoded by first converting to the more compact four value form: + ``` - `start`: 20 - `size`: 8 - `target`: 100 - `depth<<1+lasti`: 6 +start: 20 +size: 8 +target: 100 +depth<<1+lasti: 6 ``` which is then encoded as: + ``` - 148 (MSB + 20 for start) - 8 (size) - 65 (Extend bit + 1) - 36 (Remainder of target, 100 == (1<<6)+36) - 6 +148 (MSB + 20 for start) +8 (size) +65 (Extend bit + 1) +36 (Remainder of target, 100 == (1<<6)+36) +6 ``` for a total of five bytes. diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index 06dc8f0702c3d9..2598873ca98479 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -27,6 +27,7 @@ objects, so are not allocated in the per-thread stack. See `PyGenObject` in ## Layout Each activation record is laid out as: + * Specials * Locals * Stack diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 272a0834cbfe24..9e01a5864e33f8 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -1,4 +1,3 @@ - Garbage collector design ======================== @@ -117,7 +116,7 @@ general, the collection of all objects tracked by GC is partitioned into disjoin doubly linked list. Between collections, objects are partitioned into "generations", reflecting how often they've survived collection attempts. During collections, the generation(s) being collected are further partitioned into, for example, sets of reachable and unreachable objects. Doubly linked lists -support moving an object from one partition to another, adding a new object, removing an object +support moving an object from one partition to another, adding a new object, removing an object entirely (objects tracked by GC are most often reclaimed by the refcounting system when GC isn't running at all!), and merging partitions, all with a small constant number of pointer updates. With care, they also support iterating over a partition while objects are being added to - and diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index d53f0f9bdff4e4..afa8b8f4bb8040 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -1,4 +1,3 @@ - Generators ========== diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index 4c10cbbed37735..ab149e43471072 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -1,4 +1,3 @@ - The bytecode interpreter ======================== diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index 348988b7c2f003..445b866fc0cb96 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -1,4 +1,3 @@ - Guide to the parser =================== @@ -444,15 +443,15 @@ How to regenerate the parser Once you have made the changes to the grammar files, to regenerate the `C` parser (the one used by the interpreter) just execute: -``` - make regen-pegen +```shell +$ make regen-pegen ``` using the `Makefile` in the main directory. If you are on Windows you can use the Visual Studio project files to regenerate the parser or to execute: -``` - ./PCbuild/build.bat --regen +```dos +PCbuild/build.bat --regen ``` The generated parser file is located at [`Parser/parser.c`](../Parser/parser.c). @@ -468,15 +467,15 @@ any modifications to this file (in order to implement new Pegen features) you wi need to regenerate the meta-parser (the parser that parses the grammar files). To do so just execute: -``` - make regen-pegen-metaparser +```shell +$ make regen-pegen-metaparser ``` If you are on Windows you can use the Visual Studio project files to regenerate the parser or to execute: -``` - ./PCbuild/build.bat --regen +```dos +PCbuild/build.bat --regen ``` @@ -516,15 +515,15 @@ be found in the [`Grammar/Tokens`](../Grammar/Tokens) file. If you change this file to add new tokens, make sure to regenerate the files by executing: -``` - make regen-token +```shell +$ make regen-token ``` If you are on Windows you can use the Visual Studio project files to regenerate the tokens or to execute: -``` - ./PCbuild/build.bat --regen +```dos +PCbuild/build.bat --regen ``` How tokens are generated and the rules governing this are completely up to the tokenizer @@ -546,8 +545,8 @@ by default** except for rules with the special marker `memo` after the rule name (and type, if present): ``` - rule_name[typr] (memo): - ... +rule_name[typr] (memo): + ... ``` By selectively turning on memoization for a handful of rules, the parser becomes @@ -593,25 +592,25 @@ are always reserved words, even in positions where they make no sense meaning in context. Trying to use a hard keyword as a variable will always fail: -``` - >>> class = 3 - File "", line 1 - class = 3 - ^ - SyntaxError: invalid syntax - >>> foo(class=3) - File "", line 1 - foo(class=3) - ^^^^^ - SyntaxError: invalid syntax +```pycon +>>> class = 3 +File "", line 1 + class = 3 + ^ +SyntaxError: invalid syntax +>>> foo(class=3) +File "", line 1 + foo(class=3) + ^^^^^ +SyntaxError: invalid syntax ``` While soft keywords don't have this limitation if used in a context other the one where they are defined as keywords: -``` - >>> match = 45 - >>> foo(match="Yeah!") +```pycon +>>> match = 45 +>>> foo(match="Yeah!") ``` The `match` and `case` keywords are soft keywords, so that they are @@ -621,21 +620,21 @@ argument names. You can get a list of all keywords defined in the grammar from Python: -``` - >>> import keyword - >>> keyword.kwlist - ['False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', - 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', - 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', - 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'] +```pycon +>>> import keyword +>>> keyword.kwlist +['False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', +'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', +'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', +'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'] ``` as well as soft keywords: -``` - >>> import keyword - >>> keyword.softkwlist - ['_', 'case', 'match'] +```pycon +>>> import keyword +>>> keyword.softkwlist +['_', 'case', 'match'] ``` > [!CAUTION] @@ -736,7 +735,7 @@ displayed when the error is reported. > rule or not. For example: ``` - $ 42 + $ 42 ``` should trigger the syntax error in the `$` character. If your rule is not correctly defined this @@ -744,7 +743,7 @@ won't happen. As another example, suppose that you try to define a rule to match `print` statements in order to create a better error message and you define it as: ``` - invalid_print: "print" expression +invalid_print: "print" expression ``` This will **seem** to work because the parser will correctly parse `print(something)` because it is valid @@ -756,7 +755,7 @@ will be reported there instead of the `$` character. Generating AST objects ---------------------- -The output of the C parser used by CPython, which is generated from the +The output of the C parser used by CPython, which is generated from the [grammar file](../Grammar/python.gram), is a Python AST object (using C structures). This means that the actions in the grammar file generate AST objects when they succeed. Constructing these objects can be quite cumbersome @@ -798,7 +797,7 @@ Check the contents of these files to know which is the best place for new tests, depending on the nature of the new feature you are adding. Tests for the parser generator itself can be found in the -[test_peg_generator](../Lib/test_peg_generator) directory. +[test_peg_generator](../Lib/test/test_peg_generator) directory. Debugging generated parsers @@ -816,15 +815,15 @@ For this reason it is a good idea to experiment first by generating a Python parser. To do this, you can go to the [Tools/peg_generator](../Tools/peg_generator) directory on the CPython repository and manually call the parser generator by executing: -``` - $ python -m pegen python +```shell +$ python -m pegen python ``` This will generate a file called `parse.py` in the same directory that you can use to parse some input: -``` - $ python parse.py file_with_source_code_to_test.py +```shell +$ python parse.py file_with_source_code_to_test.py ``` As the generated `parse.py` file is just Python code, you can modify it @@ -848,8 +847,8 @@ can be a bit hard to understand at first. To activate verbose mode you can add the `-d` flag when executing Python: -``` - $ python -d file_to_test.py +```shell +$ python -d file_to_test.py ``` This will print **a lot** of output to `stderr` so it is probably better to dump @@ -857,7 +856,7 @@ it to a file for further analysis. The output consists of trace lines with the following structure:: ``` - ('>'|'-'|'+'|'!') []: ... + ('>'|'-'|'+'|'!') []: ... ``` Every line is indented by a different amount (``) depending on how diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index e0d20632516142..26a5197c6e70f3 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -2,6 +2,7 @@ *Interned* strings are conceptually part of an interpreter-global *set* of interned strings, meaning that: + - no two interned strings have the same content (across an interpreter); - two interned strings can be safely compared using pointer equality (Python `is`). @@ -61,6 +62,7 @@ if it's interned and mortal it needs extra processing in The converse is not true: interned strings can be mortal. For mortal interned strings: + - the 2 references from the interned dict (key & value) are excluded from their refcount - the deallocator (`unicode_dealloc`) removes the string from the interned dict @@ -90,6 +92,7 @@ modify in place. The functions take ownership of (“steal”) the reference to their argument, and update the argument with a *new* reference. This means: + - They're “reference neutral”. - They must not be called with a borrowed reference. From 7ea523f47cdb4cf512a1e2ae1f93f5d19a48945d Mon Sep 17 00:00:00 2001 From: Zhikang Yan <2951256653@qq.com> Date: Mon, 2 Dec 2024 03:29:27 +0800 Subject: [PATCH 064/427] gh-126899: Add `**kw` to `tkinter.Misc.after` and `tkinter.Misc.after_idle` (#126900) --------- Co-authored-by: Serhiy Storchaka --- Doc/whatsnew/3.14.rst | 8 ++++++++ Lib/test/test_tkinter/test_misc.py | 20 +++++++++++++++---- Lib/tkinter/__init__.py | 8 ++++---- Misc/ACKS | 1 + ...-11-16-10-52-48.gh-issue-126899.GFnfBt.rst | 2 ++ 5 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 869a47c1261293..f9322da3d4fbb0 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -576,6 +576,14 @@ sys from other interpreters than the one it's called in. +tkinter +------- + +* Make tkinter widget methods :meth:`!after` and :meth:`!after_idle` accept + arguments passed by keyword. + (Contributed by Zhikang Yan in :gh:`126899`.) + + unicodedata ----------- diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 579ce2af9fa0bf..475edcbd5338a7 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -123,9 +123,9 @@ def test_tk_setPalette(self): def test_after(self): root = self.root - def callback(start=0, step=1): + def callback(start=0, step=1, *, end=0): nonlocal count - count = start + step + count = start + step + end # Without function, sleeps for ms. self.assertIsNone(root.after(1)) @@ -161,12 +161,18 @@ def callback(start=0, step=1): root.update() # Process all pending events. self.assertEqual(count, 53) + # Set up with callback with keyword args. + count = 0 + timer1 = root.after(0, callback, 42, step=11, end=1) + root.update() # Process all pending events. + self.assertEqual(count, 54) + def test_after_idle(self): root = self.root - def callback(start=0, step=1): + def callback(start=0, step=1, *, end=0): nonlocal count - count = start + step + count = start + step + end # Set up with callback with no args. count = 0 @@ -193,6 +199,12 @@ def callback(start=0, step=1): with self.assertRaises(tkinter.TclError): root.tk.call(script) + # Set up with callback with keyword args. + count = 0 + idle1 = root.after_idle(callback, 42, step=11, end=1) + root.update() # Process all pending events. + self.assertEqual(count, 54) + def test_after_cancel(self): root = self.root diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index dd7b3e138f4236..bfec04bb6c1e6e 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -847,7 +847,7 @@ def tk_focusPrev(self): if not name: return None return self._nametowidget(name) - def after(self, ms, func=None, *args): + def after(self, ms, func=None, *args, **kw): """Call function once after given time. MS specifies the time in milliseconds. FUNC gives the @@ -861,7 +861,7 @@ def after(self, ms, func=None, *args): else: def callit(): try: - func(*args) + func(*args, **kw) finally: try: self.deletecommand(name) @@ -875,13 +875,13 @@ def callit(): name = self._register(callit) return self.tk.call('after', ms, name) - def after_idle(self, func, *args): + def after_idle(self, func, *args, **kw): """Call FUNC once if the Tcl main loop has no event to process. Return an identifier to cancel the scheduling with after_cancel.""" - return self.after('idle', func, *args) + return self.after('idle', func, *args, **kw) def after_cancel(self, id): """Cancel scheduling of function identified with ID. diff --git a/Misc/ACKS b/Misc/ACKS index cd34846574b304..fc4b83a0e2b823 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -2079,6 +2079,7 @@ Arnon Yaari Alakshendra Yadav Hirokazu Yamamoto Masayuki Yamamoto +Zhikang Yan Jingchen Ye Ka-Ping Yee Chi Hsuan Yen diff --git a/Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst b/Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst new file mode 100644 index 00000000000000..c1a0ed6438135d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst @@ -0,0 +1,2 @@ +Make tkinter widget methods :meth:`!after` and :meth:`!after_idle` accept +arguments passed by keyword. From e2713409cff5b71b1176b0e3fa63dae447548672 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 2 Dec 2024 10:38:17 +0900 Subject: [PATCH 065/427] gh-115999: Add partial free-thread specialization for BINARY_SUBSCR (gh-127227) --- Include/internal/pycore_list.h | 3 + Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_metadata.h | 2 +- Lib/test/test_dis.py | 21 ----- Lib/test/test_opcache.py | 107 ++++++++++++++++------ Objects/listobject.c | 6 ++ Python/bytecodes.c | 10 +- Python/executor_cases.c.h | 11 +++ Python/generated_cases.c.h | 12 ++- Python/specialize.c | 25 +++-- 10 files changed, 128 insertions(+), 71 deletions(-) diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 2c666f9be4bd79..f03e484f5ef8b0 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -10,6 +10,9 @@ extern "C" { PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); +// _PyList_GetItemRef should be used only when the object is known as a list +// because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does. +extern PyObject* _PyList_GetItemRef(PyListObject *, Py_ssize_t i); #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 5ce172856e1b19..d63c8df8ca6690 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1952,7 +1952,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1c1f478c3833c8..1825bb3a5abc80 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -84,7 +84,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SUBSCR_STR_INT] = HAS_DEOPT_FLAG, [_BINARY_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, [_BINARY_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index f26411ace8fa73..55890e58ed4bae 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1260,27 +1260,6 @@ def test_super_instructions(self): got = self.get_disassembly(load_test, adaptive=True) self.do_disassembly_compare(got, dis_load_test_quickened_code) - @cpython_only - @requires_specialization - def test_binary_subscr_specialize(self): - binary_subscr_quicken = """\ - 0 RESUME_CHECK 0 - - 1 LOAD_NAME 0 (a) - LOAD_SMALL_INT 0 - %s - RETURN_VALUE -""" - co_list = compile('a[0]', "", "eval") - self.code_quicken(lambda: exec(co_list, {}, {'a': [0]})) - got = self.get_disassembly(co_list, adaptive=True) - self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_LIST_INT") - - co_dict = compile('a[0]', "", "eval") - self.code_quicken(lambda: exec(co_dict, {}, {'a': {0: '1'}})) - got = self.get_disassembly(co_dict, adaptive=True) - self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_DICT") - @cpython_only @requires_specialization def test_load_attr_specialize(self): diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 1a6eac236009c3..b989b21cd9b3a9 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -617,7 +617,7 @@ def write(items): opname = "BINARY_SUBSCR_GETITEM" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_binary_subscr_list_int(self): def get_items(): items = [] @@ -1023,7 +1023,7 @@ def write(items): opname = "STORE_ATTR_WITH_HINT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_store_subscr_list_int(self): def get_items(): items = [] @@ -1229,48 +1229,48 @@ class TestSpecializer(TestBase): @cpython_only @requires_specialization_ft def test_binary_op(self): - def f(): + def binary_op_add_int(): for _ in range(100): a, b = 1, 2 c = a + b self.assertEqual(c, 3) - f() - self.assert_specialized(f, "BINARY_OP_ADD_INT") - self.assert_no_opcode(f, "BINARY_OP") + binary_op_add_int() + self.assert_specialized(binary_op_add_int, "BINARY_OP_ADD_INT") + self.assert_no_opcode(binary_op_add_int, "BINARY_OP") - def g(): + def binary_op_add_unicode(): for _ in range(100): a, b = "foo", "bar" c = a + b self.assertEqual(c, "foobar") - g() - self.assert_specialized(g, "BINARY_OP_ADD_UNICODE") - self.assert_no_opcode(g, "BINARY_OP") + binary_op_add_unicode() + self.assert_specialized(binary_op_add_unicode, "BINARY_OP_ADD_UNICODE") + self.assert_no_opcode(binary_op_add_unicode, "BINARY_OP") @cpython_only @requires_specialization_ft def test_contain_op(self): - def f(): + def contains_op_dict(): for _ in range(100): a, b = 1, {1: 2, 2: 5} self.assertTrue(a in b) self.assertFalse(3 in b) - f() - self.assert_specialized(f, "CONTAINS_OP_DICT") - self.assert_no_opcode(f, "CONTAINS_OP") + contains_op_dict() + self.assert_specialized(contains_op_dict, "CONTAINS_OP_DICT") + self.assert_no_opcode(contains_op_dict, "CONTAINS_OP") - def g(): + def contains_op_set(): for _ in range(100): a, b = 1, {1, 2} self.assertTrue(a in b) self.assertFalse(3 in b) - g() - self.assert_specialized(g, "CONTAINS_OP_SET") - self.assert_no_opcode(g, "CONTAINS_OP") + contains_op_set() + self.assert_specialized(contains_op_set, "CONTAINS_OP_SET") + self.assert_no_opcode(contains_op_set, "CONTAINS_OP") @cpython_only @requires_specialization_ft @@ -1342,34 +1342,81 @@ def to_bool_str(): @cpython_only @requires_specialization_ft def test_unpack_sequence(self): - def f(): + def unpack_sequence_two_tuple(): for _ in range(100): a, b = 1, 2 self.assertEqual(a, 1) self.assertEqual(b, 2) - f() - self.assert_specialized(f, "UNPACK_SEQUENCE_TWO_TUPLE") - self.assert_no_opcode(f, "UNPACK_SEQUENCE") + unpack_sequence_two_tuple() + self.assert_specialized(unpack_sequence_two_tuple, + "UNPACK_SEQUENCE_TWO_TUPLE") + self.assert_no_opcode(unpack_sequence_two_tuple, "UNPACK_SEQUENCE") - def g(): + def unpack_sequence_tuple(): for _ in range(100): a, = 1, self.assertEqual(a, 1) - g() - self.assert_specialized(g, "UNPACK_SEQUENCE_TUPLE") - self.assert_no_opcode(g, "UNPACK_SEQUENCE") + unpack_sequence_tuple() + self.assert_specialized(unpack_sequence_tuple, "UNPACK_SEQUENCE_TUPLE") + self.assert_no_opcode(unpack_sequence_tuple, "UNPACK_SEQUENCE") - def x(): + def unpack_sequence_list(): for _ in range(100): a, b = [1, 2] self.assertEqual(a, 1) self.assertEqual(b, 2) - x() - self.assert_specialized(x, "UNPACK_SEQUENCE_LIST") - self.assert_no_opcode(x, "UNPACK_SEQUENCE") + unpack_sequence_list() + self.assert_specialized(unpack_sequence_list, "UNPACK_SEQUENCE_LIST") + self.assert_no_opcode(unpack_sequence_list, "UNPACK_SEQUENCE") + + @cpython_only + @requires_specialization_ft + def test_binary_subscr(self): + def binary_subscr_list_int(): + for _ in range(100): + a = [1, 2, 3] + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_list_int() + self.assert_specialized(binary_subscr_list_int, + "BINARY_SUBSCR_LIST_INT") + self.assert_no_opcode(binary_subscr_list_int, "BINARY_SUBSCR") + + def binary_subscr_tuple_int(): + for _ in range(100): + a = (1, 2, 3) + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_tuple_int() + self.assert_specialized(binary_subscr_tuple_int, + "BINARY_SUBSCR_TUPLE_INT") + self.assert_no_opcode(binary_subscr_tuple_int, "BINARY_SUBSCR") + + def binary_subscr_dict(): + for _ in range(100): + a = {1: 2, 2: 3} + self.assertEqual(a[1], 2) + self.assertEqual(a[2], 3) + + binary_subscr_dict() + self.assert_specialized(binary_subscr_dict, "BINARY_SUBSCR_DICT") + self.assert_no_opcode(binary_subscr_dict, "BINARY_SUBSCR") + + def binary_subscr_str_int(): + for _ in range(100): + a = "foobar" + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_str_int() + self.assert_specialized(binary_subscr_str_int, "BINARY_SUBSCR_STR_INT") + self.assert_no_opcode(binary_subscr_str_int, "BINARY_SUBSCR") + if __name__ == "__main__": unittest.main() diff --git a/Objects/listobject.c b/Objects/listobject.c index bb0040cbe9f272..4b24f4a428e18b 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -391,6 +391,12 @@ PyList_GetItemRef(PyObject *op, Py_ssize_t i) return item; } +PyObject * +_PyList_GetItemRef(PyListObject *list, Py_ssize_t i) +{ + return list_get_item_ref(list, i); +} + int PyList_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a14b32b8108be8..c07ec42ec68f8b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -704,7 +704,7 @@ dummy_func( }; specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT assert(frame->stackpointer == NULL); if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -713,7 +713,7 @@ dummy_func( } OPCODE_DEFERRED_INC(BINARY_SUBSCR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_BINARY_SUBSCR, (container, sub -- res)) { @@ -790,11 +790,17 @@ dummy_func( // Deopt unless 0 <= sub < PyList_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; +#ifdef Py_GIL_DISABLED + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + DEOPT_IF(res_o == NULL); + STAT_INC(BINARY_SUBSCR, hit); +#else DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); +#endif PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); DEAD(sub_st); PyStackRef_CLOSE(list_st); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d46412a193332b..c91257b06cad11 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -981,6 +981,16 @@ JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_SUBSCR, hit); + #else if (index >= PyList_GET_SIZE(list)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -989,6 +999,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); + #endif PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c9a5132269398c..45bcc4242af9d7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -433,7 +433,7 @@ container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT assert(frame->stackpointer == NULL); if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -444,7 +444,7 @@ } OPCODE_DEFERRED_INC(BINARY_SUBSCR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _BINARY_SUBSCR { @@ -577,11 +577,19 @@ // Deopt unless 0 <= sub < PyList_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + #else DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); + #endif PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); diff --git a/Python/specialize.c b/Python/specialize.c index 172dae7d374602..d03310de782fe7 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1717,15 +1717,15 @@ _Py_Specialize_BinarySubscr( PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[BINARY_SUBSCR] == INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1); PyTypeObject *container_type = Py_TYPE(container); + uint8_t specialized_op; if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - instr->op.code = BINARY_SUBSCR_LIST_INT; + specialized_op = BINARY_SUBSCR_LIST_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1738,7 +1738,7 @@ _Py_Specialize_BinarySubscr( if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - instr->op.code = BINARY_SUBSCR_TUPLE_INT; + specialized_op = BINARY_SUBSCR_TUPLE_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1751,7 +1751,7 @@ _Py_Specialize_BinarySubscr( if (container_type == &PyUnicode_Type) { if (PyLong_CheckExact(sub)) { if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - instr->op.code = BINARY_SUBSCR_STR_INT; + specialized_op = BINARY_SUBSCR_STR_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1762,9 +1762,10 @@ _Py_Specialize_BinarySubscr( goto fail; } if (container_type == &PyDict_Type) { - instr->op.code = BINARY_SUBSCR_DICT; + specialized_op = BINARY_SUBSCR_DICT; goto success; } +#ifndef Py_GIL_DISABLED PyTypeObject *cls = Py_TYPE(container); PyObject *descriptor = _PyType_Lookup(cls, &_Py_ID(__getitem__)); if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { @@ -1797,21 +1798,17 @@ _Py_Specialize_BinarySubscr( // struct _specialization_cache): ht->_spec_cache.getitem = descriptor; ht->_spec_cache.getitem_version = version; - instr->op.code = BINARY_SUBSCR_GETITEM; + specialized_op = BINARY_SUBSCR_GETITEM; goto success; } +#endif // Py_GIL_DISABLED SPECIALIZATION_FAIL(BINARY_SUBSCR, binary_subscr_fail_kind(container_type, sub)); fail: - STAT_INC(BINARY_SUBSCR, failure); - assert(!PyErr_Occurred()); - instr->op.code = BINARY_SUBSCR; - cache->counter = adaptive_counter_backoff(cache->counter); + unspecialize(instr); return; success: - STAT_INC(BINARY_SUBSCR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + specialize(instr, specialized_op); } From 2950bc50af8fc2539e64731359bfb39b335a614d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Mon, 2 Dec 2024 07:12:36 +0000 Subject: [PATCH 066/427] GH-127429: fix sysconfig data generation on cross-builds (#127430) --- .github/workflows/jit.yml | 2 +- Lib/sysconfig/__init__.py | 49 ++++++++++++++----- Lib/sysconfig/__main__.py | 7 ++- ...-11-29-23-02-43.gh-issue-127429.dQf2w4.rst | 3 ++ configure | 6 +-- configure.ac | 2 +- 6 files changed, 49 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 7dbbe71b2131e7..4ef543d7369734 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -104,7 +104,7 @@ jobs: # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - name: Native macOS if: runner.os == 'macOS' diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index ee52700b51fd07..ad86609016e478 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -318,14 +318,22 @@ def get_default_scheme(): def get_makefile_filename(): """Return the path of the Makefile.""" + + # GH-127429: When cross-compiling, use the Makefile from the target, instead of the host Python. + if cross_base := os.environ.get('_PYTHON_PROJECT_BASE'): + return os.path.join(cross_base, 'Makefile') + if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") + if hasattr(sys, 'abiflags'): config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' else: config_dir_name = 'config' + if hasattr(sys.implementation, '_multiarch'): config_dir_name += f'-{sys.implementation._multiarch}' + return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') @@ -464,27 +472,44 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): def _init_config_vars(): global _CONFIG_VARS _CONFIG_VARS = {} + + prefix = _PREFIX + exec_prefix = _EXEC_PREFIX + base_prefix = _BASE_PREFIX + base_exec_prefix = _BASE_EXEC_PREFIX + + try: + abiflags = sys.abiflags + except AttributeError: + abiflags = '' + + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + # If we are cross-compiling, load the prefixes from the Makefile instead. + if '_PYTHON_PROJECT_BASE' in os.environ: + prefix = _CONFIG_VARS['prefix'] + exec_prefix = _CONFIG_VARS['exec_prefix'] + base_prefix = _CONFIG_VARS['prefix'] + base_exec_prefix = _CONFIG_VARS['exec_prefix'] + abiflags = _CONFIG_VARS['ABIFLAGS'] + # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX + _CONFIG_VARS['prefix'] = prefix + _CONFIG_VARS['exec_prefix'] = exec_prefix _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX + _CONFIG_VARS['installed_base'] = base_prefix + _CONFIG_VARS['base'] = prefix + _CONFIG_VARS['installed_platbase'] = base_exec_prefix + _CONFIG_VARS['platbase'] = exec_prefix _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['platlibdir'] = sys.platlibdir _CONFIG_VARS['implementation'] = _get_implementation() _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' + _CONFIG_VARS['abiflags'] = abiflags try: _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') except AttributeError: @@ -493,8 +518,6 @@ def _init_config_vars(): if os.name == 'nt': _init_non_posix(_CONFIG_VARS) _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the # init function to enable using 'get_config_var' in diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 5660a6c5105b9f..10728c709e1811 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -7,6 +7,7 @@ _PYTHON_BUILD, _get_sysconfigdata_name, get_config_h_filename, + get_config_var, get_config_vars, get_default_scheme, get_makefile_filename, @@ -161,7 +162,7 @@ def _print_config_dict(d, stream): def _get_pybuilddir(): pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): + if get_config_var('Py_DEBUG') == '1': pybuilddir += '-pydebug' return pybuilddir @@ -229,11 +230,15 @@ def _generate_posix_vars(): f.write('build_time_vars = ') _print_config_dict(vars, stream=f) + print(f'Written {destfile}') + # Write a JSON file with the output of sysconfig.get_config_vars jsonfile = os.path.join(pybuilddir, _get_json_data_name()) with open(jsonfile, 'w') as f: json.dump(get_config_vars(), f, indent=2) + print(f'Written {jsonfile}') + # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) diff --git a/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst new file mode 100644 index 00000000000000..708c1a6437d812 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst @@ -0,0 +1,3 @@ +Fixed bug where, on cross-builds, the :mod:`sysconfig` POSIX data was being +generated with the host Python's ``Makefile``. The data is now generated from +current build's ``Makefile``. diff --git a/configure b/configure index 84b74ac3584bcd..4e4043260ed2df 100755 --- a/configure +++ b/configure @@ -944,8 +944,8 @@ AR LINK_PYTHON_OBJS LINK_PYTHON_DEPS LIBRARY_DEPS -NODE HOSTRUNNER +NODE STATIC_LIBPYTHON GNULD EXPORTSFROM @@ -1147,7 +1147,6 @@ LDFLAGS LIBS CPPFLAGS CPP -HOSTRUNNER PROFILE_TASK BOLT_INSTRUMENT_FLAGS BOLT_APPLY_FLAGS @@ -1968,7 +1967,6 @@ Some influential environment variables: CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if you have headers in a nonstandard directory CPP C preprocessor - HOSTRUNNER Program to run CPython for the host platform PROFILE_TASK Python args for PGO generation task BOLT_INSTRUMENT_FLAGS @@ -7622,9 +7620,9 @@ if test "$cross_compiling" = yes; then RUNSHARED= fi +# HOSTRUNNER - Program to run CPython for the host platform { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 printf %s "checking HOSTRUNNER... " >&6; } - if test -z "$HOSTRUNNER" then case $ac_sys_system in #( diff --git a/configure.ac b/configure.ac index 8fa6cb60900ad1..4cfced10432491 100644 --- a/configure.ac +++ b/configure.ac @@ -1609,8 +1609,8 @@ if test "$cross_compiling" = yes; then RUNSHARED= fi +# HOSTRUNNER - Program to run CPython for the host platform AC_MSG_CHECKING([HOSTRUNNER]) -AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) if test -z "$HOSTRUNNER" then AS_CASE([$ac_sys_system], From 1f8267b85dda655282922ba20df90d0ac6bea634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 2 Dec 2024 09:21:00 +0100 Subject: [PATCH 067/427] gh-127443: Fix some entries in `Doc/data/refcounts.dat` (#127451) Fix incorrect entries in `Doc/data/refcounts.dat` --- Doc/data/refcounts.dat | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 6bfcc191b2270b..3f49c88c3cc028 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -180,7 +180,7 @@ PyCapsule_IsValid:const char*:name:: PyCapsule_New:PyObject*::+1: PyCapsule_New:void*:pointer:: PyCapsule_New:const char *:name:: -PyCapsule_New::void (* destructor)(PyObject* ):: +PyCapsule_New:void (*)(PyObject *):destructor:: PyCapsule_SetContext:int::: PyCapsule_SetContext:PyObject*:self:0: @@ -349,11 +349,11 @@ PyComplex_CheckExact:int::: PyComplex_CheckExact:PyObject*:p:0: PyComplex_FromCComplex:PyObject*::+1: -PyComplex_FromCComplex::Py_complex v:: +PyComplex_FromCComplex:Py_complex:v:: PyComplex_FromDoubles:PyObject*::+1: -PyComplex_FromDoubles::double real:: -PyComplex_FromDoubles::double imag:: +PyComplex_FromDoubles:double:real:: +PyComplex_FromDoubles:double:imag:: PyComplex_ImagAsDouble:double::: PyComplex_ImagAsDouble:PyObject*:op:0: @@ -620,7 +620,9 @@ PyErr_GetExcInfo:PyObject**:pvalue:+1: PyErr_GetExcInfo:PyObject**:ptraceback:+1: PyErr_GetRaisedException:PyObject*::+1: -PyErr_SetRaisedException:::: + +PyErr_SetRaisedException:void::: +PyErr_SetRaisedException:PyObject *:exc:0:stolen PyErr_GivenExceptionMatches:int::: PyErr_GivenExceptionMatches:PyObject*:given:0: @@ -640,9 +642,9 @@ PyErr_NewExceptionWithDoc:PyObject*:dict:0: PyErr_NoMemory:PyObject*::null: PyErr_NormalizeException:void::: -PyErr_NormalizeException:PyObject**:exc::??? -PyErr_NormalizeException:PyObject**:val::??? -PyErr_NormalizeException:PyObject**:tb::??? +PyErr_NormalizeException:PyObject**:exc:+1:??? +PyErr_NormalizeException:PyObject**:val:+1:??? +PyErr_NormalizeException:PyObject**:tb:+1:??? PyErr_Occurred:PyObject*::0: @@ -1314,7 +1316,7 @@ PyMapping_GetItemString:const char*:key:: PyMapping_HasKey:int::: PyMapping_HasKey:PyObject*:o:0: -PyMapping_HasKey:PyObject*:key:: +PyMapping_HasKey:PyObject*:key:0: PyMapping_HasKeyString:int::: PyMapping_HasKeyString:PyObject*:o:0: @@ -1474,7 +1476,7 @@ PyModule_GetState:void*::: PyModule_GetState:PyObject*:module:0: PyModule_New:PyObject*::+1: -PyModule_New::char* name:: +PyModule_New:char*:name:: PyModule_NewObject:PyObject*::+1: PyModule_NewObject:PyObject*:name:+1: @@ -1484,7 +1486,7 @@ PyModule_SetDocString:PyObject*:module:0: PyModule_SetDocString:const char*:docstring:: PyModuleDef_Init:PyObject*::0: -PyModuleDef_Init:PyModuleDef*:def:0: +PyModuleDef_Init:PyModuleDef*:def:: PyNumber_Absolute:PyObject*::+1: PyNumber_Absolute:PyObject*:o:0: @@ -1984,10 +1986,10 @@ PyRun_StringFlags:PyObject*:locals:0: PyRun_StringFlags:PyCompilerFlags*:flags:: PySeqIter_Check:int::: -PySeqIter_Check::op:: +PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: -PySeqIter_New:PyObject*:seq:: +PySeqIter_New:PyObject*:seq:0: PySequence_Check:int::: PySequence_Check:PyObject*:o:0: @@ -2421,7 +2423,7 @@ PyUnicode_GET_LENGTH:PyObject*:o:0: PyUnicode_KIND:int::: PyUnicode_KIND:PyObject*:o:0: -PyUnicode_MAX_CHAR_VALUE:::: +PyUnicode_MAX_CHAR_VALUE:Py_UCS4::: PyUnicode_MAX_CHAR_VALUE:PyObject*:o:0: Py_UNICODE_ISALNUM:int::: @@ -2488,7 +2490,7 @@ PyUnicode_FromWideChar:const wchar_t*:w:: PyUnicode_FromWideChar:Py_ssize_t:size:: PyUnicode_AsWideChar:Py_ssize_t::: -PyUnicode_AsWideChar:PyObject*:*unicode:0: +PyUnicode_AsWideChar:PyObject*:unicode:0: PyUnicode_AsWideChar:wchar_t*:w:: PyUnicode_AsWideChar:Py_ssize_t:size:: @@ -2541,7 +2543,7 @@ PyUnicode_AsUTF8String:PyObject*:unicode:0: PyUnicode_AsUTF8AndSize:const char*::: PyUnicode_AsUTF8AndSize:PyObject*:unicode:0: -PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:0: +PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:: PyUnicode_AsUTF8:const char*::: PyUnicode_AsUTF8:PyObject*:unicode:0: @@ -2864,13 +2866,13 @@ PyUnicodeDecodeError_SetStart:PyObject*:exc:0: PyUnicodeDecodeError_SetStart:Py_ssize_t:start:: PyWeakref_Check:int::: -PyWeakref_Check:PyObject*:ob:: +PyWeakref_Check:PyObject*:ob:0: PyWeakref_CheckProxy:int::: -PyWeakref_CheckProxy:PyObject*:ob:: +PyWeakref_CheckProxy:PyObject*:ob:0: PyWeakref_CheckRef:int::: -PyWeakref_CheckRef:PyObject*:ob:: +PyWeakref_CheckRef:PyObject*:ob:0: PyWeakref_GET_OBJECT:PyObject*::0: PyWeakref_GET_OBJECT:PyObject*:ref:0: From 2a373da7700cf928e0a5ce3998d19351a3565df4 Mon Sep 17 00:00:00 2001 From: CF Bolz-Tereick Date: Mon, 2 Dec 2024 11:11:28 +0100 Subject: [PATCH 068/427] add missing gc_collect() calls in sqlite3 tests (#127446) --- Lib/test/test_sqlite3/test_regression.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index db4e13222da9da..50cced3891d13a 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -433,6 +433,7 @@ def test_table_lock_cursor_dealloc(self): con.commit() cur = con.execute("select t from t") del cur + support.gc_collect() con.execute("drop table t") con.commit() @@ -448,6 +449,7 @@ def dup(v): con.create_function("dup", 1, dup) cur = con.execute("select dup(t) from t") del cur + support.gc_collect() con.execute("drop table t") con.commit() From a8dd821d5b25b42c0adeae6642e9b3f9228580f9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 2 Dec 2024 10:12:17 +0000 Subject: [PATCH 069/427] GH-126491: GC: Mark objects reachable from roots before doing cycle collection (GH-127110) * Mark almost all reachable objects before doing collection phase * Add stats for objects marked * Visit new frames before each increment * Update docs * Clearer calculation of work to do. --- Include/cpython/pystats.h | 2 + Include/internal/pycore_frame.h | 3 + Include/internal/pycore_gc.h | 10 +- Include/internal/pycore_object.h | 4 +- Include/internal/pycore_runtime_init.h | 1 + InternalDocs/garbage_collector.md | 39 ++ Lib/test/libregrtest/refleak.py | 2 +- Lib/test/test_gc.py | 24 +- ...-11-21-16-13-52.gh-issue-126491.0YvL94.rst | 4 + Modules/_testinternalcapi.c | 6 + Python/ceval.c | 1 + Python/gc.c | 355 +++++++++++++----- Python/specialize.c | 2 + Tools/scripts/summarize_stats.py | 5 +- 14 files changed, 355 insertions(+), 103 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index f1ca54839fbc38..2ae48002d720e9 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -99,6 +99,8 @@ typedef struct _gc_stats { uint64_t collections; uint64_t object_visits; uint64_t objects_collected; + uint64_t objects_transitively_reachable; + uint64_t objects_not_transitively_reachable; } GCStats; typedef struct _uop_stats { diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 8c0100390d036e..b786c5f49e9831 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -75,6 +75,7 @@ typedef struct _PyInterpreterFrame { _PyStackRef *stackpointer; uint16_t return_offset; /* Only relevant during a function call */ char owner; + char visited; /* Locals and stack */ _PyStackRef localsplus[1]; } _PyInterpreterFrame; @@ -207,6 +208,7 @@ _PyFrame_Initialize( #endif frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; + frame->visited = 0; for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = PyStackRef_NULL; @@ -389,6 +391,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->instr_ptr = _PyCode_CODE(code); #endif frame->owner = FRAME_OWNED_BY_THREAD; + frame->visited = 0; frame->return_offset = 0; #ifdef Py_GIL_DISABLED diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 479fe10d00066d..4ff34bf8ead7d0 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -10,11 +10,11 @@ extern "C" { /* GC information is stored BEFORE the object structure. */ typedef struct { - // Pointer to next object in the list. + // Tagged pointer to next object in the list. // 0 means the object is not tracked uintptr_t _gc_next; - // Pointer to previous object in the list. + // Tagged pointer to previous object in the list. // Lowest two bits are used for flags documented later. uintptr_t _gc_prev; } PyGC_Head; @@ -284,6 +284,11 @@ struct gc_generation_stats { Py_ssize_t uncollectable; }; +enum _GCPhase { + GC_PHASE_MARK = 0, + GC_PHASE_COLLECT = 1 +}; + struct _gc_runtime_state { /* List of objects that still need to be cleaned up, singly linked * via their gc headers' gc_prev pointers. */ @@ -311,6 +316,7 @@ struct _gc_runtime_state { Py_ssize_t work_to_do; /* Which of the old spaces is the visited space */ int visited_space; + int phase; #ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 34d835a7f84ee7..c52ed8f14707ba 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -471,8 +471,8 @@ static inline void _PyObject_GC_TRACK( PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - /* Young objects will be moved into the visited space during GC, so set the bit here */ - gc->_gc_next = ((uintptr_t)generation0) | (uintptr_t)interp->gc.visited_space; + uintptr_t not_visited = 1 ^ interp->gc.visited_space; + gc->_gc_next = ((uintptr_t)generation0) | not_visited; generation0->_gc_prev = (uintptr_t)gc; #endif } diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 9f6748945bab36..1260b957ce9482 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -137,6 +137,7 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 0, }, \ }, \ .work_to_do = -5000, \ + .phase = GC_PHASE_MARK, \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 9e01a5864e33f8..08db080a200ea4 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -477,6 +477,45 @@ specifically in a generation by calling `gc.collect(generation=NUM)`. ``` +Optimization: visiting reachable objects +======================================== + +An object cannot be garbage if it can be reached. + +To avoid having to identify reference cycles across the whole heap, we can +reduce the amount of work done considerably by first moving most reachable objects +to the `visited` space. Empirically, most reachable objects can be reached from a +small set of global objects and local variables. +This step does much less work per object, so reduces the time spent +performing garbage collection by at least half. + +> [!NOTE] +> Objects that are not determined to be reachable by this pass are not necessarily +> unreachable. We still need to perform the main algorithm to determine which objects +> are actually unreachable. +We use the same technique of forming a transitive closure as the incremental +collector does to find reachable objects, seeding the list with some global +objects and the currently executing frames. + +This phase moves objects to the `visited` space, as follows: + +1. All objects directly referred to by any builtin class, the `sys` module, the `builtins` +module and all objects directly referred to from stack frames are added to a working +set of reachable objects. +2. Until this working set is empty: + 1. Pop an object from the set and move it to the `visited` space + 2. For each object directly reachable from that object: + * If it is not already in `visited` space and it is a GC object, + add it to the working set + + +Before each increment of collection is performed, the stacks are scanned +to check for any new stack frames that have been created since the last +increment. All objects directly referred to from those stack frames are +added to the working set. +Then the above algorithm is repeated, starting from step 2. + + Optimization: reusing fields to save memory =========================================== diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index e783475cc7a36b..d0d1c8cdc9a11b 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -123,9 +123,9 @@ def get_pooled_int(value): xml_filename = 'refleak-xml.tmp' result = None dash_R_cleanup(fs, ps, pic, zdc, abcs) - support.gc_collect() for i in rep_range: + support.gc_collect() current = refleak_helper._hunting_for_refleaks refleak_helper._hunting_for_refleaks = True try: diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 0372815b9bfd27..b5140057a69d36 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -31,6 +31,11 @@ def __new__(cls, *args, **kwargs): return C ContainerNoGC = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + ### Support code ############################################################################### @@ -1130,6 +1135,7 @@ def setUp(self): def tearDown(self): gc.disable() + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") @requires_gil_enabled("Free threading does not support incremental GC") # Use small increments to emulate longer running process in a shorter time @gc_threshold(200, 10) @@ -1167,20 +1173,15 @@ def make_ll(depth): enabled = gc.isenabled() gc.enable() olds = [] + initial_heap_size = _testinternalcapi.get_tracked_heap_size() for i in range(20_000): newhead = make_ll(20) count += 20 newhead.surprise = head olds.append(newhead) if len(olds) == 20: - stats = gc.get_stats() - young = stats[0] - incremental = stats[1] - old = stats[2] - collected = young['collected'] + incremental['collected'] + old['collected'] - count += CORRECTION - live = count - collected - self.assertLess(live, 25000) + new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size + self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations") del olds[:] if not enabled: gc.disable() @@ -1322,7 +1323,8 @@ def test_refcount_errors(self): from test.support import gc_collect, SuppressCrashReport a = [1, 2, 3] - b = [a] + b = [a, a] + a.append(b) # Avoid coredump when Py_FatalError() calls abort() SuppressCrashReport().__enter__() @@ -1332,6 +1334,8 @@ def test_refcount_errors(self): # (to avoid deallocating it): import ctypes ctypes.pythonapi.Py_DecRef(ctypes.py_object(a)) + del a + del b # The garbage collector should now have a fatal error # when it reaches the broken object @@ -1360,7 +1364,7 @@ def test_refcount_errors(self): self.assertRegex(stderr, br'object type name: list') self.assertRegex(stderr, - br'object repr : \[1, 2, 3\]') + br'object repr : \[1, 2, 3, \[\[...\], \[...\]\]\]') class GCTogglingTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst new file mode 100644 index 00000000000000..9ef2b8dc33ed0f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst @@ -0,0 +1,4 @@ +Add a marking phase to the GC. All objects that can be transitively reached +from builtin modules or the stacks are marked as reachable before cycle +detection. This reduces the amount of work done by the GC by approximately +half. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index a925191d479bd6..1bb71a3e80b39d 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2076,6 +2076,11 @@ has_deferred_refcount(PyObject *self, PyObject *op) return PyBool_FromLong(_PyObject_HasDeferredRefcount(op)); } +static PyObject * +get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size); +} static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, @@ -2174,6 +2179,7 @@ static PyMethodDef module_functions[] = { {"get_static_builtin_types", get_static_builtin_types, METH_NOARGS}, {"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS}, {"has_deferred_refcount", has_deferred_refcount, METH_O}, + {"get_tracked_heap_size", get_tracked_heap_size, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/ceval.c b/Python/ceval.c index eba0f233a81ef3..f9514a6bf25c1b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -818,6 +818,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; entry_frame.stackpointer = entry_frame.localsplus; entry_frame.owner = FRAME_OWNED_BY_CSTACK; + entry_frame.visited = 0; entry_frame.return_offset = 0; /* Push frame */ entry_frame.previous = tstate->current_frame; diff --git a/Python/gc.c b/Python/gc.c index 63adecf0e05114..5b9588c8741b97 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -106,7 +106,7 @@ gc_old_space(PyGC_Head *g) } static inline int -flip_old_space(int space) +other_space(int space) { assert(space == 0 || space == 1); return space ^ _PyGC_NEXT_MASK_OLD_SPACE_1; @@ -430,24 +430,32 @@ validate_list(PyGC_Head *head, enum flagstates flags) #endif #ifdef GC_EXTRA_DEBUG + + static void -validate_old(GCState *gcstate) +gc_list_validate_space(PyGC_Head *head, int space) { + PyGC_Head *gc = GC_NEXT(head); + while (gc != head) { + assert(gc_old_space(gc) == space); + gc = GC_NEXT(gc); + } +} + +static void +validate_spaces(GCState *gcstate) { + int visited = gcstate->visited_space; + int not_visited = other_space(visited); + gc_list_validate_space(&gcstate->young.head, not_visited); for (int space = 0; space < 2; space++) { - PyGC_Head *head = &gcstate->old[space].head; - PyGC_Head *gc = GC_NEXT(head); - while (gc != head) { - PyGC_Head *next = GC_NEXT(gc); - assert(gc_old_space(gc) == space); - gc = next; - } + gc_list_validate_space(&gcstate->old[space].head, space); } + gc_list_validate_space(&gcstate->permanent_generation.head, visited); } static void validate_consistent_old_space(PyGC_Head *head) { - PyGC_Head *prev = head; PyGC_Head *gc = GC_NEXT(head); if (gc == head) { return; @@ -457,23 +465,13 @@ validate_consistent_old_space(PyGC_Head *head) PyGC_Head *truenext = GC_NEXT(gc); assert(truenext != NULL); assert(gc_old_space(gc) == old_space); - prev = gc; gc = truenext; } - assert(prev == GC_PREV(head)); } -static void -gc_list_validate_space(PyGC_Head *head, int space) { - PyGC_Head *gc = GC_NEXT(head); - while (gc != head) { - assert(gc_old_space(gc) == space); - gc = GC_NEXT(gc); - } -} #else -#define validate_old(g) do{}while(0) +#define validate_spaces(g) do{}while(0) #define validate_consistent_old_space(l) do{}while(0) #define gc_list_validate_space(l, s) do{}while(0) #endif @@ -494,7 +492,7 @@ update_refs(PyGC_Head *containers) next = GC_NEXT(gc); PyObject *op = FROM_GC(gc); if (_Py_IsImmortal(op)) { - gc_list_move(gc, &get_gc_state()->permanent_generation.head); + _PyObject_GC_UNTRACK(op); gc = next; continue; } @@ -733,13 +731,25 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) unreachable->_gc_next &= _PyGC_PREV_MASK; } +/* In theory, all tuples should be younger than the +* objects they refer to, as tuples are immortal. +* Therefore, untracking tuples in oldest-first order in the +* young generation before promoting them should have tracked +* all the tuples that can be untracked. +* +* Unfortunately, the C API allows tuples to be created +* and then filled in. So this won't untrack all tuples +* that can be untracked. It should untrack most of them +* and is much faster than a more complex approach that +* would untrack all relevant tuples. +*/ static void untrack_tuples(PyGC_Head *head) { - PyGC_Head *next, *gc = GC_NEXT(head); + PyGC_Head *gc = GC_NEXT(head); while (gc != head) { PyObject *op = FROM_GC(gc); - next = GC_NEXT(gc); + PyGC_Head *next = GC_NEXT(gc); if (PyTuple_CheckExact(op)) { _PyTuple_MaybeUntrack(op); } @@ -1293,8 +1303,10 @@ gc_collect_young(PyThreadState *tstate, struct gc_collection_stats *stats) { GCState *gcstate = &tstate->interp->gc; + validate_spaces(gcstate); PyGC_Head *young = &gcstate->young.head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + untrack_tuples(young); GC_STAT_ADD(0, collections, 1); #ifdef Py_STATS { @@ -1308,39 +1320,21 @@ gc_collect_young(PyThreadState *tstate, PyGC_Head survivors; gc_list_init(&survivors); + gc_list_set_space(young, gcstate->visited_space); gc_collect_region(tstate, young, &survivors, stats); - Py_ssize_t survivor_count = 0; - if (gcstate->visited_space) { - /* objects in visited space have bit set, so we set it here */ - survivor_count = gc_list_set_space(&survivors, 1); - } - else { - PyGC_Head *gc; - for (gc = GC_NEXT(&survivors); gc != &survivors; gc = GC_NEXT(gc)) { -#ifdef GC_DEBUG - assert(gc_old_space(gc) == 0); -#endif - survivor_count++; - } - } - (void)survivor_count; // Silence compiler warning gc_list_merge(&survivors, visited); - validate_old(gcstate); + validate_spaces(gcstate); gcstate->young.count = 0; gcstate->old[gcstate->visited_space].count++; - Py_ssize_t scale_factor = gcstate->old[0].threshold; - if (scale_factor < 1) { - scale_factor = 1; - } - gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; add_stats(gcstate, 0, stats); + validate_spaces(gcstate); } #ifndef NDEBUG static inline int IS_IN_VISITED(PyGC_Head *gc, int visited_space) { - assert(visited_space == 0 || flip_old_space(visited_space) == 0); + assert(visited_space == 0 || other_space(visited_space) == 0); return gc_old_space(gc) == visited_space; } #endif @@ -1348,7 +1342,7 @@ IS_IN_VISITED(PyGC_Head *gc, int visited_space) struct container_and_flag { PyGC_Head *container; int visited_space; - uintptr_t size; + intptr_t size; }; /* A traversal callback for adding to container) */ @@ -1371,7 +1365,7 @@ visit_add_to_container(PyObject *op, void *arg) return 0; } -static uintptr_t +static intptr_t expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate) { struct container_and_flag arg = { @@ -1385,6 +1379,7 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat * have been marked as visited */ assert(IS_IN_VISITED(gc, gcstate->visited_space)); PyObject *op = FROM_GC(gc); + assert(_PyObject_GC_IS_TRACKED(op)); if (_Py_IsImmortal(op)) { PyGC_Head *next = GC_NEXT(gc); gc_list_move(gc, &get_gc_state()->permanent_generation.head); @@ -1402,22 +1397,191 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat /* Do bookkeeping for a completed GC cycle */ static void -completed_cycle(GCState *gcstate) -{ -#ifdef Py_DEBUG - PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; - assert(gc_list_is_empty(not_visited)); -#endif - gcstate->visited_space = flip_old_space(gcstate->visited_space); - /* Make sure all young objects have old space bit set correctly */ - PyGC_Head *young = &gcstate->young.head; - PyGC_Head *gc = GC_NEXT(young); - while (gc != young) { - PyGC_Head *next = GC_NEXT(gc); - gc_set_old_space(gc, gcstate->visited_space); - gc = next; +completed_scavenge(GCState *gcstate) +{ + /* We must observe two invariants: + * 1. Members of the permanent generation must be marked visited. + * 2. We cannot touch members of the permanent generation. */ + int visited; + if (gc_list_is_empty(&gcstate->permanent_generation.head)) { + /* Permanent generation is empty so we can flip spaces bit */ + int not_visited = gcstate->visited_space; + visited = other_space(not_visited); + gcstate->visited_space = visited; + /* Make sure all objects have visited bit set correctly */ + gc_list_set_space(&gcstate->young.head, not_visited); } + else { + /* We must move the objects from visited to pending space. */ + visited = gcstate->visited_space; + int not_visited = other_space(visited); + assert(gc_list_is_empty(&gcstate->old[not_visited].head)); + gc_list_merge(&gcstate->old[visited].head, &gcstate->old[not_visited].head); + gc_list_set_space(&gcstate->old[not_visited].head, not_visited); + } + assert(gc_list_is_empty(&gcstate->old[visited].head)); gcstate->work_to_do = 0; + gcstate->phase = GC_PHASE_MARK; +} + +static intptr_t +move_to_reachable(PyObject *op, PyGC_Head *reachable, int visited_space) +{ + if (op != NULL && !_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited_space) { + gc_flip_old_space(gc); + gc_list_move(gc, reachable); + return 1; + } + } + return 0; +} + +static intptr_t +mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space) +{ + // Transitively traverse all objects from reachable, until empty + struct container_and_flag arg = { + .container = reachable, + .visited_space = visited_space, + .size = 0 + }; + while (!gc_list_is_empty(reachable)) { + PyGC_Head *gc = _PyGCHead_NEXT(reachable); + assert(gc_old_space(gc) == visited_space); + gc_list_move(gc, visited); + PyObject *op = FROM_GC(gc); + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_add_to_container, + &arg); + } + gc_list_validate_space(visited, visited_space); + return arg.size; +} + +static intptr_t +mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start) +{ + PyGC_Head reachable; + gc_list_init(&reachable); + Py_ssize_t objects_marked = 0; + // Move all objects on stacks to reachable + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); + while (ts) { + _PyInterpreterFrame *frame = ts->current_frame; + while (frame) { + if (frame->owner == FRAME_OWNED_BY_CSTACK) { + frame = frame->previous; + continue; + } + _PyStackRef *locals = frame->localsplus; + _PyStackRef *sp = frame->stackpointer; + objects_marked += move_to_reachable(frame->f_locals, &reachable, visited_space); + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + objects_marked += move_to_reachable(func, &reachable, visited_space); + while (sp > locals) { + sp--; + if (PyStackRef_IsNull(*sp)) { + continue; + } + PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); + if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited_space) { + gc_flip_old_space(gc); + objects_marked++; + gc_list_move(gc, &reachable); + } + } + } + if (!start && frame->visited) { + // If this frame has already been visited, then the lower frames + // will have already been visited and will not have changed + break; + } + frame->visited = 1; + frame = frame->previous; + } + HEAD_LOCK(runtime); + ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); + } + objects_marked += mark_all_reachable(&reachable, visited, visited_space); + assert(gc_list_is_empty(&reachable)); + return objects_marked; +} + +static intptr_t +mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space) +{ + PyGC_Head reachable; + gc_list_init(&reachable); + Py_ssize_t objects_marked = 0; + objects_marked += move_to_reachable(interp->sysdict, &reachable, visited_space); + objects_marked += move_to_reachable(interp->builtins, &reachable, visited_space); + objects_marked += move_to_reachable(interp->dict, &reachable, visited_space); + struct types_state *types = &interp->types; + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { + objects_marked += move_to_reachable(types->builtins.initialized[i].tp_dict, &reachable, visited_space); + objects_marked += move_to_reachable(types->builtins.initialized[i].tp_subclasses, &reachable, visited_space); + } + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_dict, &reachable, visited_space); + objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_subclasses, &reachable, visited_space); + } + objects_marked += mark_all_reachable(&reachable, visited, visited_space); + assert(gc_list_is_empty(&reachable)); + return objects_marked; +} + +static intptr_t +mark_at_start(PyThreadState *tstate) +{ + // TO DO -- Make this incremental + GCState *gcstate = &tstate->interp->gc; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space); + objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true); + gcstate->work_to_do -= objects_marked; + gcstate->phase = GC_PHASE_COLLECT; + validate_spaces(gcstate); + return objects_marked; +} + +static intptr_t +assess_work_to_do(GCState *gcstate) +{ + /* The amount of work we want to do depends on three things. + * 1. The number of new objects created + * 2. The growth in heap size since the last collection + * 3. The heap size (up to the number of new objects, to avoid quadratic effects) + * + * For a steady state heap, the amount of work to do is three times the number + * of new objects added to the heap. This ensures that we stay ahead in the + * worst case of all new objects being garbage. + * + * This could be improved by tracking survival rates, but it is still a + * large improvement on the non-marking approach. + */ + intptr_t scale_factor = gcstate->old[0].threshold; + if (scale_factor < 2) { + scale_factor = 2; + } + intptr_t new_objects = gcstate->young.count; + intptr_t max_heap_fraction = new_objects*3/2; + intptr_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; + if (heap_fraction > max_heap_fraction) { + heap_fraction = max_heap_fraction; + } + gcstate->young.count = 0; + return new_objects + heap_fraction; } static void @@ -1425,18 +1589,30 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) { GC_STAT_ADD(1, collections, 1); GCState *gcstate = &tstate->interp->gc; + gcstate->work_to_do += assess_work_to_do(gcstate); + untrack_tuples(&gcstate->young.head); + if (gcstate->phase == GC_PHASE_MARK) { + Py_ssize_t objects_marked = mark_at_start(tstate); + GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); + gcstate->work_to_do -= objects_marked; + validate_spaces(gcstate); + return; + } PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; PyGC_Head increment; gc_list_init(&increment); - Py_ssize_t scale_factor = gcstate->old[0].threshold; - if (scale_factor < 1) { - scale_factor = 1; - } + int scale_factor = gcstate->old[0].threshold; + if (scale_factor < 2) { + scale_factor = 2; + } + intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false); + GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); + gcstate->work_to_do -= objects_marked; + gc_list_set_space(&gcstate->young.head, gcstate->visited_space); gc_list_merge(&gcstate->young.head, &increment); - gcstate->young.count = 0; gc_list_validate_space(&increment, gcstate->visited_space); - Py_ssize_t increment_size = 0; + Py_ssize_t increment_size = gc_list_size(&increment); while (increment_size < gcstate->work_to_do) { if (gc_list_is_empty(not_visited)) { break; @@ -1444,54 +1620,56 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) PyGC_Head *gc = _PyGCHead_NEXT(not_visited); gc_list_move(gc, &increment); increment_size++; + assert(!_Py_IsImmortal(FROM_GC(gc))); gc_set_old_space(gc, gcstate->visited_space); increment_size += expand_region_transitively_reachable(&increment, gc, gcstate); } + GC_STAT_ADD(1, objects_not_transitively_reachable, increment_size); validate_list(&increment, collecting_clear_unreachable_clear); gc_list_validate_space(&increment, gcstate->visited_space); PyGC_Head survivors; gc_list_init(&survivors); gc_collect_region(tstate, &increment, &survivors, stats); - gc_list_validate_space(&survivors, gcstate->visited_space); gc_list_merge(&survivors, visited); assert(gc_list_is_empty(&increment)); gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; gcstate->work_to_do -= increment_size; - validate_old(gcstate); add_stats(gcstate, 1, stats); if (gc_list_is_empty(not_visited)) { - completed_cycle(gcstate); + completed_scavenge(gcstate); } + validate_spaces(gcstate); } - static void gc_collect_full(PyThreadState *tstate, struct gc_collection_stats *stats) { GC_STAT_ADD(2, collections, 1); GCState *gcstate = &tstate->interp->gc; - validate_old(gcstate); + validate_spaces(gcstate); PyGC_Head *young = &gcstate->young.head; PyGC_Head *pending = &gcstate->old[gcstate->visited_space^1].head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + untrack_tuples(young); /* merge all generations into visited */ - gc_list_validate_space(young, gcstate->visited_space); - gc_list_set_space(pending, gcstate->visited_space); gc_list_merge(young, pending); + gc_list_validate_space(pending, 1-gcstate->visited_space); + gc_list_set_space(pending, gcstate->visited_space); gcstate->young.count = 0; gc_list_merge(pending, visited); + validate_spaces(gcstate); gc_collect_region(tstate, visited, visited, stats); + validate_spaces(gcstate); gcstate->young.count = 0; gcstate->old[0].count = 0; gcstate->old[1].count = 0; - - gcstate->work_to_do = - gcstate->young.threshold * 2; + completed_scavenge(gcstate); _PyGC_ClearAllFreeLists(tstate->interp); - validate_old(gcstate); + validate_spaces(gcstate); add_stats(gcstate, 2, stats); } @@ -1733,20 +1911,23 @@ void _PyGC_Freeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; - /* The permanent_generation has its old space bit set to zero */ - if (gcstate->visited_space) { - gc_list_set_space(&gcstate->young.head, 0); - } + /* The permanent_generation must be visited */ + gc_list_set_space(&gcstate->young.head, gcstate->visited_space); gc_list_merge(&gcstate->young.head, &gcstate->permanent_generation.head); gcstate->young.count = 0; PyGC_Head*old0 = &gcstate->old[0].head; PyGC_Head*old1 = &gcstate->old[1].head; + if (gcstate->visited_space) { + gc_list_set_space(old0, 1); + } + else { + gc_list_set_space(old1, 0); + } gc_list_merge(old0, &gcstate->permanent_generation.head); gcstate->old[0].count = 0; - gc_list_set_space(old1, 0); gc_list_merge(old1, &gcstate->permanent_generation.head); gcstate->old[1].count = 0; - validate_old(gcstate); + validate_spaces(gcstate); } void @@ -1754,8 +1935,8 @@ _PyGC_Unfreeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; gc_list_merge(&gcstate->permanent_generation.head, - &gcstate->old[0].head); - validate_old(gcstate); + &gcstate->old[gcstate->visited_space].head); + validate_spaces(gcstate); } Py_ssize_t @@ -1860,7 +2041,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) _Py_stats->object_stats.object_visits = 0; } #endif - validate_old(gcstate); + validate_spaces(gcstate); _Py_atomic_store_int(&gcstate->collecting, 0); return stats.uncollectable + stats.collected; } diff --git a/Python/specialize.c b/Python/specialize.c index d03310de782fe7..504eef4f448429 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -231,6 +231,8 @@ print_gc_stats(FILE *out, GCStats *stats) fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); + fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); + fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); } } diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index abfdea78253760..bc7ccfe33e777d 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -1118,6 +1118,8 @@ def calc_gc_stats(stats: Stats) -> Rows: Count(gen["collections"]), Count(gen["objects collected"]), Count(gen["object visits"]), + Count(gen["objects reachable from roots"]), + Count(gen["objects not reachable from roots"]), ) for (i, gen) in enumerate(gc_stats) ] @@ -1127,7 +1129,8 @@ def calc_gc_stats(stats: Stats) -> Rows: "GC collections and effectiveness", [ Table( - ("Generation:", "Collections:", "Objects collected:", "Object visits:"), + ("Generation:", "Collections:", "Objects collected:", "Object visits:", + "Reachable from roots:", "Not reachable from roots:"), calc_gc_stats, ) ], From bf21e2160d1dc6869fb230b90a23ab030835395b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 2 Dec 2024 11:14:47 +0100 Subject: [PATCH 070/427] Fix Unicode encode_wstr_utf8() (#127420) Raise RuntimeError instead of RuntimeWarning. --- Objects/unicodeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 33fa21d4c7d1bf..463da06445984b 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -16158,7 +16158,7 @@ encode_wstr_utf8(wchar_t *wstr, char **str, const char *name) int res; res = _Py_EncodeUTF8Ex(wstr, str, NULL, NULL, 1, _Py_ERROR_STRICT); if (res == -2) { - PyErr_Format(PyExc_RuntimeWarning, "cannot encode %s", name); + PyErr_Format(PyExc_RuntimeError, "cannot encode %s", name); return -1; } if (res < 0) { From c112de1da2d18e3b5c2ea30b0e409f18e574efd8 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 2 Dec 2024 07:50:34 -0500 Subject: [PATCH 071/427] gh-126890: Restore stripped `ssl` docstrings (GH-127281) --- Modules/_ssl.c | 63 ++++++++++++++++++++++++++++++--------- Modules/clinic/_ssl.c.h | 65 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 14 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index e5b8bf21002ea5..59c414f9ce1ceb 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2162,11 +2162,17 @@ _ssl__SSLSocket_compression_impl(PySSLSocket *self) @critical_section @getter _ssl._SSLSocket.context + +This changes the context associated with the SSLSocket. + +This is typically used from within a callback function set by the sni_callback +on the SSLContext to change the certificate information associated with the +SSLSocket before the cryptographic exchange handshake messages. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_context_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=d23e82f72f32e3d7 input=25aa82e4d9fa344a]*/ +/*[clinic end generated code: output=d23e82f72f32e3d7 input=7cbb97407c2ace30]*/ { return Py_NewRef(self->ctx); } @@ -2233,11 +2239,15 @@ _ssl__SSLSocket_server_hostname_get_impl(PySSLSocket *self) @critical_section @getter _ssl._SSLSocket.owner + +The Python-level owner of this object. + +Passed as "self" in servername callback. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_owner_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=1f278cb930382927 input=bc2861ff3cf1402e]*/ +/*[clinic end generated code: output=1f278cb930382927 input=978a8382d9c25c92]*/ { if (self->owner == NULL) { Py_RETURN_NONE; @@ -2851,11 +2861,13 @@ _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) @critical_section @getter _ssl._SSLSocket.session + +The underlying SSLSession object. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_session_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=a5cd5755b35da670 input=b9792df9255a9f63]*/ +/*[clinic end generated code: output=a5cd5755b35da670 input=d427318604244bf8]*/ { /* get_session can return sessions from a server-side connection, * it does not check for handshake done or client socket. */ @@ -3657,11 +3669,13 @@ _ssl__SSLContext_maximum_version_set_impl(PySSLContext *self, @critical_section @getter _ssl._SSLContext.num_tickets + +Control the number of TLSv1.3 session tickets. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_num_tickets_get_impl(PySSLContext *self) -/*[clinic end generated code: output=3d06d016318846c9 input=1dee26d75163c073]*/ +/*[clinic end generated code: output=3d06d016318846c9 input=1e2599a2e22564ff]*/ { // Clinic seems to be misbehaving when the comment is wrapped with in directive #if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) @@ -3705,11 +3719,13 @@ _ssl__SSLContext_num_tickets_set_impl(PySSLContext *self, PyObject *value) @critical_section @getter _ssl._SSLContext.security_level + +The current security level. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_security_level_get_impl(PySSLContext *self) -/*[clinic end generated code: output=56ece09e6a9572d0 input=a0416598e07c3183]*/ +/*[clinic end generated code: output=56ece09e6a9572d0 input=2bdeecb57bb86e3f]*/ { PyObject *res = PyLong_FromLong(SSL_CTX_get_security_level(self->ctx)); return res; @@ -4666,11 +4682,18 @@ _servername_callback(SSL *s, int *al, void *args) @critical_section @getter _ssl._SSLContext.sni_callback + +Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension. + +If the argument is None then the callback is disabled. The method is called +with the SSLSocket, the server name as a string, and the SSLContext object. + +See RFC 6066 for details of the SNI extension. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) -/*[clinic end generated code: output=961e6575cdfaf036 input=22dd28c31fdc4318]*/ +/*[clinic end generated code: output=961e6575cdfaf036 input=9b2473c5e984cfe6]*/ { PyObject *cb = self->set_sni_cb; if (cb == NULL) { @@ -5243,11 +5266,13 @@ memory_bio_dealloc(PySSLMemoryBIO *self) @critical_section @getter _ssl.MemoryBIO.pending + +The number of bytes pending in the memory BIO. [clinic start generated code]*/ static PyObject * _ssl_MemoryBIO_pending_get_impl(PySSLMemoryBIO *self) -/*[clinic end generated code: output=19236a32a51ac8ff input=c0b6d14eba107f6a]*/ +/*[clinic end generated code: output=19236a32a51ac8ff input=02d9063d8ac31732]*/ { size_t res = BIO_ctrl_pending(self->bio); return PyLong_FromSize_t(res); @@ -5257,11 +5282,13 @@ _ssl_MemoryBIO_pending_get_impl(PySSLMemoryBIO *self) @critical_section @getter _ssl.MemoryBIO.eof + +Whether the memory BIO is at EOF. [clinic start generated code]*/ static PyObject * _ssl_MemoryBIO_eof_get_impl(PySSLMemoryBIO *self) -/*[clinic end generated code: output=c255a9ea16e31b92 input=0f5c6be69752e04c]*/ +/*[clinic end generated code: output=c255a9ea16e31b92 input=c6ecc12c4509de1f]*/ { size_t pending = BIO_ctrl_pending(self->bio); return PyBool_FromLong((pending == 0) && self->eof_written); @@ -5502,11 +5529,13 @@ PySSLSession_clear(PySSLSession *self) @critical_section @getter _ssl.SSLSession.time + +Session creation time (seconds since epoch). [clinic start generated code]*/ static PyObject * _ssl_SSLSession_time_get_impl(PySSLSession *self) -/*[clinic end generated code: output=4b887b9299de9be4 input=8d1e4afd09103279]*/ +/*[clinic end generated code: output=4b887b9299de9be4 input=67f2325284450ae2]*/ { #if OPENSSL_VERSION_NUMBER >= 0x30300000L return _PyLong_FromTime_t(SSL_SESSION_get_time_ex(self->session)); @@ -5519,11 +5548,13 @@ _ssl_SSLSession_time_get_impl(PySSLSession *self) @critical_section @getter _ssl.SSLSession.timeout + +Session timeout (delta in seconds). [clinic start generated code]*/ static PyObject * _ssl_SSLSession_timeout_get_impl(PySSLSession *self) -/*[clinic end generated code: output=82339c148ab2f7d1 input=ae5e84a9d85df60d]*/ +/*[clinic end generated code: output=82339c148ab2f7d1 input=cd17c2b087c442f2]*/ { long timeout = SSL_SESSION_get_timeout(self->session); PyObject *res = PyLong_FromLong(timeout); @@ -5534,11 +5565,13 @@ _ssl_SSLSession_timeout_get_impl(PySSLSession *self) @critical_section @getter _ssl.SSLSession.ticket_lifetime_hint + +Ticket life time hint. [clinic start generated code]*/ static PyObject * _ssl_SSLSession_ticket_lifetime_hint_get_impl(PySSLSession *self) -/*[clinic end generated code: output=c8b6db498136c275 input=d0e06942ddd8d07f]*/ +/*[clinic end generated code: output=c8b6db498136c275 input=f0e2df50961a7806]*/ { unsigned long hint = SSL_SESSION_get_ticket_lifetime_hint(self->session); return PyLong_FromUnsignedLong(hint); @@ -5548,11 +5581,13 @@ _ssl_SSLSession_ticket_lifetime_hint_get_impl(PySSLSession *self) @critical_section @getter _ssl.SSLSession.id + +Session ID. [clinic start generated code]*/ static PyObject * _ssl_SSLSession_id_get_impl(PySSLSession *self) -/*[clinic end generated code: output=c532fb96b10c5adf input=e7322372cf6325dd]*/ +/*[clinic end generated code: output=c532fb96b10c5adf input=0a379e64312b776d]*/ { const unsigned char *id; @@ -5565,11 +5600,13 @@ _ssl_SSLSession_id_get_impl(PySSLSession *self) @critical_section @getter _ssl.SSLSession.has_ticket + +Does the session contain a ticket? [clinic start generated code]*/ static PyObject * _ssl_SSLSession_has_ticket_get_impl(PySSLSession *self) -/*[clinic end generated code: output=aa3ccfc40b10b96d input=1a48ae8955fa9601]*/ +/*[clinic end generated code: output=aa3ccfc40b10b96d input=fa475555f53a5086]*/ { int res = SSL_SESSION_has_ticket(self->session); return res ? Py_True : Py_False; diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 957f5ced3a2cee..1ff85e32ffe5a0 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -258,6 +258,14 @@ _ssl__SSLSocket_compression(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) return _ssl__SSLSocket_compression_impl(self); } +PyDoc_STRVAR(_ssl__SSLSocket_context__doc__, +"This changes the context associated with the SSLSocket.\n" +"\n" +"This is typically used from within a callback function set by the sni_callback\n" +"on the SSLContext to change the certificate information associated with the\n" +"SSLSocket before the cryptographic exchange handshake messages."); +#define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ + #if !defined(_ssl__SSLSocket_context_DOCSTR) # define _ssl__SSLSocket_context_DOCSTR NULL #endif @@ -366,6 +374,12 @@ _ssl__SSLSocket_server_hostname_get(PySSLSocket *self, void *Py_UNUSED(context)) return return_value; } +PyDoc_STRVAR(_ssl__SSLSocket_owner__doc__, +"The Python-level owner of this object.\n" +"\n" +"Passed as \"self\" in servername callback."); +#define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ + #if !defined(_ssl__SSLSocket_owner_DOCSTR) # define _ssl__SSLSocket_owner_DOCSTR NULL #endif @@ -652,6 +666,10 @@ _ssl__SSLSocket_verify_client_post_handshake(PySSLSocket *self, PyObject *Py_UNU return return_value; } +PyDoc_STRVAR(_ssl__SSLSocket_session__doc__, +"The underlying SSLSession object."); +#define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ + #if !defined(_ssl__SSLSocket_session_DOCSTR) # define _ssl__SSLSocket_session_DOCSTR NULL #endif @@ -1057,6 +1075,10 @@ _ssl__SSLContext_maximum_version_set(PySSLContext *self, PyObject *value, void * return return_value; } +PyDoc_STRVAR(_ssl__SSLContext_num_tickets__doc__, +"Control the number of TLSv1.3 session tickets."); +#define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ + #if !defined(_ssl__SSLContext_num_tickets_DOCSTR) # define _ssl__SSLContext_num_tickets_DOCSTR NULL #endif @@ -1107,6 +1129,10 @@ _ssl__SSLContext_num_tickets_set(PySSLContext *self, PyObject *value, void *Py_U return return_value; } +PyDoc_STRVAR(_ssl__SSLContext_security_level__doc__, +"The current security level."); +#define _ssl__SSLContext_security_level_DOCSTR _ssl__SSLContext_security_level__doc__ + #if !defined(_ssl__SSLContext_security_level_DOCSTR) # define _ssl__SSLContext_security_level_DOCSTR NULL #endif @@ -1745,6 +1771,15 @@ _ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name) return return_value; } +PyDoc_STRVAR(_ssl__SSLContext_sni_callback__doc__, +"Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n" +"\n" +"If the argument is None then the callback is disabled. The method is called\n" +"with the SSLSocket, the server name as a string, and the SSLContext object.\n" +"\n" +"See RFC 6066 for details of the SNI extension."); +#define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ + #if !defined(_ssl__SSLContext_sni_callback_DOCSTR) # define _ssl__SSLContext_sni_callback_DOCSTR NULL #endif @@ -2063,6 +2098,10 @@ _ssl_MemoryBIO(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(_ssl_MemoryBIO_pending__doc__, +"The number of bytes pending in the memory BIO."); +#define _ssl_MemoryBIO_pending_DOCSTR _ssl_MemoryBIO_pending__doc__ + #if !defined(_ssl_MemoryBIO_pending_DOCSTR) # define _ssl_MemoryBIO_pending_DOCSTR NULL #endif @@ -2088,6 +2127,10 @@ _ssl_MemoryBIO_pending_get(PySSLMemoryBIO *self, void *Py_UNUSED(context)) return return_value; } +PyDoc_STRVAR(_ssl_MemoryBIO_eof__doc__, +"Whether the memory BIO is at EOF."); +#define _ssl_MemoryBIO_eof_DOCSTR _ssl_MemoryBIO_eof__doc__ + #if !defined(_ssl_MemoryBIO_eof_DOCSTR) # define _ssl_MemoryBIO_eof_DOCSTR NULL #endif @@ -2217,6 +2260,10 @@ _ssl_MemoryBIO_write_eof(PySSLMemoryBIO *self, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(_ssl_SSLSession_time__doc__, +"Session creation time (seconds since epoch)."); +#define _ssl_SSLSession_time_DOCSTR _ssl_SSLSession_time__doc__ + #if !defined(_ssl_SSLSession_time_DOCSTR) # define _ssl_SSLSession_time_DOCSTR NULL #endif @@ -2242,6 +2289,10 @@ _ssl_SSLSession_time_get(PySSLSession *self, void *Py_UNUSED(context)) return return_value; } +PyDoc_STRVAR(_ssl_SSLSession_timeout__doc__, +"Session timeout (delta in seconds)."); +#define _ssl_SSLSession_timeout_DOCSTR _ssl_SSLSession_timeout__doc__ + #if !defined(_ssl_SSLSession_timeout_DOCSTR) # define _ssl_SSLSession_timeout_DOCSTR NULL #endif @@ -2267,6 +2318,10 @@ _ssl_SSLSession_timeout_get(PySSLSession *self, void *Py_UNUSED(context)) return return_value; } +PyDoc_STRVAR(_ssl_SSLSession_ticket_lifetime_hint__doc__, +"Ticket life time hint."); +#define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR _ssl_SSLSession_ticket_lifetime_hint__doc__ + #if !defined(_ssl_SSLSession_ticket_lifetime_hint_DOCSTR) # define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR NULL #endif @@ -2292,6 +2347,10 @@ _ssl_SSLSession_ticket_lifetime_hint_get(PySSLSession *self, void *Py_UNUSED(con return return_value; } +PyDoc_STRVAR(_ssl_SSLSession_id__doc__, +"Session ID."); +#define _ssl_SSLSession_id_DOCSTR _ssl_SSLSession_id__doc__ + #if !defined(_ssl_SSLSession_id_DOCSTR) # define _ssl_SSLSession_id_DOCSTR NULL #endif @@ -2317,6 +2376,10 @@ _ssl_SSLSession_id_get(PySSLSession *self, void *Py_UNUSED(context)) return return_value; } +PyDoc_STRVAR(_ssl_SSLSession_has_ticket__doc__, +"Does the session contain a ticket?"); +#define _ssl_SSLSession_has_ticket_DOCSTR _ssl_SSLSession_has_ticket__doc__ + #if !defined(_ssl_SSLSession_has_ticket_DOCSTR) # define _ssl_SSLSession_has_ticket_DOCSTR NULL #endif @@ -2767,4 +2830,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=44ab066d21277ee5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=654d6d7af659f6cd input=a9049054013a1b77]*/ From 930ba0ce605eee9e3b992fa368b00a3f2b7dc4c1 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 2 Dec 2024 16:14:40 +0300 Subject: [PATCH 072/427] gh-126618: fix repr(itertools.count(sys.maxsize)) (#127048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_itertools.py | 23 +++++++++++++++++++ ...-11-20-08-54-11.gh-issue-126618.ef_53g.rst | 2 ++ Modules/itertoolsmodule.c | 9 +++----- 3 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index b94d688738f9e8..f0fd1d28f56f55 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -518,6 +518,15 @@ def test_count(self): self.assertEqual(next(c), -8) self.assertEqual(repr(count(10.25)), 'count(10.25)') self.assertEqual(repr(count(10.0)), 'count(10.0)') + + self.assertEqual(repr(count(maxsize)), f'count({maxsize})') + c = count(maxsize - 1) + self.assertEqual(repr(c), f'count({maxsize - 1})') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize})') + next(c) + self.assertEqual(repr(c), f'count({maxsize + 1})') + self.assertEqual(type(next(count(10.0))), float) for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): # Test repr @@ -578,6 +587,20 @@ def test_count_with_step(self): self.assertEqual(type(next(c)), int) self.assertEqual(type(next(c)), float) + c = count(maxsize -2, 2) + self.assertEqual(repr(c), f'count({maxsize - 2}, 2)') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize}, 2)') + next(c) + self.assertEqual(repr(c), f'count({maxsize + 2}, 2)') + + c = count(maxsize + 1, -1) + self.assertEqual(repr(c), f'count({maxsize + 1}, -1)') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize}, -1)') + next(c) + self.assertEqual(repr(c), f'count({maxsize - 1}, -1)') + @threading_helper.requires_working_threading() def test_count_threading(self, step=1): # this test verifies multithreading consistency, which is diff --git a/Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst b/Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst new file mode 100644 index 00000000000000..7a0a7b7517b70d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst @@ -0,0 +1,2 @@ +Fix the representation of :class:`itertools.count` objects when the count +value is :data:`sys.maxsize`. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 78fbdcdf77a923..3f736f0cf19968 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3235,7 +3235,7 @@ typedef struct { fast_mode: when cnt an integer < PY_SSIZE_T_MAX and no step is specified. - assert(cnt != PY_SSIZE_T_MAX && long_cnt == NULL && long_step==PyLong(1)); + assert(long_cnt == NULL && long_step==PyLong(1)); Advances with: cnt += 1 When count hits PY_SSIZE_T_MAX, switch to slow_mode. @@ -3291,9 +3291,6 @@ itertools_count_impl(PyTypeObject *type, PyObject *long_cnt, PyErr_Clear(); fast_mode = 0; } - else if (cnt == PY_SSIZE_T_MAX) { - fast_mode = 0; - } } } else { cnt = 0; @@ -3325,7 +3322,7 @@ itertools_count_impl(PyTypeObject *type, PyObject *long_cnt, else cnt = PY_SSIZE_T_MAX; - assert((cnt != PY_SSIZE_T_MAX && long_cnt == NULL && fast_mode) || + assert((long_cnt == NULL && fast_mode) || (cnt == PY_SSIZE_T_MAX && long_cnt != NULL && !fast_mode)); assert(!fast_mode || (PyLong_Check(long_step) && PyLong_AS_LONG(long_step) == 1)); @@ -3418,7 +3415,7 @@ count_next(countobject *lz) static PyObject * count_repr(countobject *lz) { - if (lz->cnt != PY_SSIZE_T_MAX) + if (lz->long_cnt == NULL) return PyUnicode_FromFormat("%s(%zd)", _PyType_Name(Py_TYPE(lz)), lz->cnt); From 31f16e427b545f66a9a45ea9dd6c933975ce0e4c Mon Sep 17 00:00:00 2001 From: Giovanni Siragusa Date: Mon, 2 Dec 2024 14:18:30 +0100 Subject: [PATCH 073/427] gh-109523: Raise a BlockingIOError if reading text from a non-blocking stream cannot immediately return bytes. (GH-122933) --- Doc/library/io.rst | 20 +++++++++++++++++++ Doc/whatsnew/3.14.rst | 9 +++++++++ Lib/_pyio.py | 5 ++++- Lib/test/test_io.py | 16 +++++++++++++++ Misc/ACKS | 1 + ...-08-12-10-15-19.gh-issue-109523.S2c3fi.rst | 1 + Modules/_io/textio.c | 6 ++++++ 7 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst diff --git a/Doc/library/io.rst b/Doc/library/io.rst index f793d7a7ef9a84..0d8cc5171d5476 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -64,6 +64,12 @@ In-memory text streams are also available as :class:`StringIO` objects:: f = io.StringIO("some initial text data") +.. note:: + + When working with a non-blocking stream, be aware that read operations on text I/O objects + might raise a :exc:`BlockingIOError` if the stream cannot perform the operation + immediately. + The text stream API is described in detail in the documentation of :class:`TextIOBase`. @@ -770,6 +776,11 @@ than raw I/O does. Read and return *size* bytes, or if *size* is not given or negative, until EOF or if the read call would block in non-blocking mode. + .. note:: + + When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` + may be raised if a read operation cannot be completed immediately. + .. method:: read1(size=-1, /) Read and return up to *size* bytes with only one call on the raw stream. @@ -779,6 +790,10 @@ than raw I/O does. .. versionchanged:: 3.7 The *size* argument is now optional. + .. note:: + + When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` + may be raised if a read operation cannot be completed immediately. .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE) @@ -1007,6 +1022,11 @@ Text I/O .. versionchanged:: 3.10 The *encoding* argument now supports the ``"locale"`` dummy encoding name. + .. note:: + + When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` + may be raised if a read operation cannot be completed immediately. + :class:`TextIOWrapper` provides these data attributes and methods in addition to those from :class:`TextIOBase` and :class:`IOBase`: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index f9322da3d4fbb0..75d027d33ccd16 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -404,6 +404,15 @@ inspect (Contributed by Zhikang Yan in :gh:`125634`.) + +io +-- + +* Reading text from a non-blocking stream with ``read`` may now raise a + :exc:`BlockingIOError` if the operation cannot immediately return bytes. + (Contributed by Giovanni Siragusa in :gh:`109523`.) + + json ---- diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 42b0aea4e2eb2e..14961c39d3541d 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -2545,9 +2545,12 @@ def read(self, size=None): size = size_index() decoder = self._decoder or self._get_decoder() if size < 0: + chunk = self.buffer.read() + if chunk is None: + raise BlockingIOError("Read returned None.") # Read everything. result = (self._get_decoded_chars() + - decoder.decode(self.buffer.read(), final=True)) + decoder.decode(chunk, final=True)) if self._snapshot is not None: self._set_decoded_chars('') self._snapshot = None diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index f1f8ce57668f3b..81c17b2731cc58 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3932,6 +3932,22 @@ def test_issue35928(self): f.write(res) self.assertEqual(res + f.readline(), 'foo\nbar\n') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_read_non_blocking(self): + import os + r, w = os.pipe() + try: + os.set_blocking(r, False) + with self.io.open(r, 'rt') as textfile: + r = None + # Nothing has been written so a non-blocking read raises a BlockingIOError exception. + with self.assertRaises(BlockingIOError): + textfile.read() + finally: + if r is not None: + os.close(r) + os.close(w) + class MemviewBytesIO(io.BytesIO): '''A BytesIO object whose read method returns memoryviews diff --git a/Misc/ACKS b/Misc/ACKS index fc4b83a0e2b823..913f7c8ecf5f1e 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1736,6 +1736,7 @@ Ng Pheng Siong Yann Sionneau George Sipe J. Sipprell +Giovanni Siragusa Ngalim Siregar Kragen Sitaker Kaartic Sivaraam diff --git a/Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst b/Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst new file mode 100644 index 00000000000000..9d6b2e0c565623 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst @@ -0,0 +1 @@ +Reading text from a non-blocking stream with ``read`` may now raise a :exc:`BlockingIOError` if the operation cannot immediately return bytes. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 0d851ee211511c..791ee070401fe5 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1992,6 +1992,12 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) if (bytes == NULL) goto fail; + if (bytes == Py_None){ + Py_DECREF(bytes); + PyErr_SetString(PyExc_BlockingIOError, "Read returned None."); + return NULL; + } + _PyIO_State *state = self->state; if (Py_IS_TYPE(self->decoder, state->PyIncrementalNewlineDecoder_Type)) decoded = _PyIncrementalNewlineDecoder_decode(self->decoder, From 3e812253ab6b2f98fc5d17bfb82947e392b0b2a2 Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Mon, 2 Dec 2024 22:51:35 +0900 Subject: [PATCH 074/427] gh-101100: Fix Sphinx warnings about list methods (#127054) --- Doc/library/collections.rst | 4 ++-- Doc/tools/.nitignore | 1 - Doc/tutorial/datastructures.rst | 10 +++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 0cc9063f153aba..5b4e445762e076 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -783,10 +783,10 @@ sequence of key-value pairs into a dictionary of lists: When each key is encountered for the first time, it is not already in the mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` -function which returns an empty :class:`list`. The :meth:`list.append` +function which returns an empty :class:`list`. The :meth:`!list.append` operation then attaches the value to the new list. When keys are encountered again, the look-up proceeds normally (returning the list for that key) and the -:meth:`list.append` operation adds another value to the list. This technique is +:meth:`!list.append` operation adds another value to the list. This technique is simpler and faster than an equivalent technique using :meth:`dict.setdefault`: >>> d = {} diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 711c0b64095bd2..39d1f5975e331c 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -71,7 +71,6 @@ Doc/library/xmlrpc.server.rst Doc/library/zlib.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst -Doc/tutorial/datastructures.rst Doc/using/windows.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 31941bc112a135..263b0c2e2815a1 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -142,8 +142,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the -top of the stack, use :meth:`~list.pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`!~list.append`. To retrieve an item from the +top of the stack, use :meth:`!~list.pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -340,7 +340,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method +value: the :keyword:`del` statement. This differs from the :meth:`!~list.pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -500,8 +500,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`~list.append` and -:meth:`~list.extend`. +assignments, slice assignments, or methods like :meth:`!~list.append` and +:meth:`!~list.extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of From 7c2bd9b2266665ff4010b6c6c175bab18e08e4f8 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 3 Dec 2024 00:14:40 +0900 Subject: [PATCH 075/427] gh-115999: Use light-weight lock for UNPACK_SEQUENCE_LIST (gh-127514) --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 13 +++---------- Python/executor_cases.c.h | 22 ++++++---------------- Python/generated_cases.c.h | 19 +++---------------- 5 files changed, 14 insertions(+), 44 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d63c8df8ca6690..81dde66a6f26c2 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -2148,7 +2148,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1825bb3a5abc80..89fce193f40bd8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -112,7 +112,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c07ec42ec68f8b..e96674c3502ef1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1438,14 +1438,9 @@ dummy_func( inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); DEOPT_IF(!PyList_CheckExact(seq_o)); - #ifdef Py_GIL_DISABLED - PyCriticalSection cs; - PyCriticalSection_Begin(&cs, seq_o); - #endif + DEOPT_IF(!LOCK_OBJECT(seq_o)); if (PyList_GET_SIZE(seq_o) != oparg) { - #ifdef Py_GIL_DISABLED - PyCriticalSection_End(&cs); - #endif + UNLOCK_OBJECT(seq_o); DEOPT_IF(true); } STAT_INC(UNPACK_SEQUENCE, hit); @@ -1453,9 +1448,7 @@ dummy_func( for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } - #ifdef Py_GIL_DISABLED - PyCriticalSection_End(&cs); - #endif + UNLOCK_OBJECT(seq_o); DECREF_INPUTS(); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c91257b06cad11..580814657608db 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1728,18 +1728,12 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - #ifdef Py_GIL_DISABLED - PyCriticalSection cs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_Begin(&cs, seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + if (!LOCK_OBJECT(seq_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } if (PyList_GET_SIZE(seq_o) != oparg) { - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_End(&cs); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + UNLOCK_OBJECT(seq_o); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -1750,11 +1744,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_End(&cs); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + UNLOCK_OBJECT(seq_o); PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 45bcc4242af9d7..e1f951558de7da 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8040,18 +8040,9 @@ values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - #ifdef Py_GIL_DISABLED - PyCriticalSection cs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_Begin(&cs, seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_End(&cs); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + UNLOCK_OBJECT(seq_o); DEOPT_IF(true, UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); @@ -8059,11 +8050,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_End(&cs); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + UNLOCK_OBJECT(seq_o); PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); From c46acd3588864e97d0e0fe37a41aa5e94ac7af51 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 2 Dec 2024 16:51:50 +0100 Subject: [PATCH 076/427] gh-126876: Fix test_socket.testLargeTimeout() for missing _testcapi (#127517) --- Lib/test/test_socket.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 7b3914f30e5f52..307d6e886c617f 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -5136,7 +5136,10 @@ def testLargeTimeout(self): # gh-126876: Check that a timeout larger than INT_MAX is replaced with # INT_MAX in the poll() code path. The following assertion must not # fail: assert(INT_MIN <= ms && ms <= INT_MAX). - large_timeout = _testcapi.INT_MAX + 1 + if _testcapi is not None: + large_timeout = _testcapi.INT_MAX + 1 + else: + large_timeout = 2147483648 # test recv() with large timeout conn, addr = self.serv.accept() @@ -5151,7 +5154,10 @@ def testLargeTimeout(self): def _testLargeTimeout(self): # test sendall() with large timeout - large_timeout = _testcapi.INT_MAX + 1 + if _testcapi is not None: + large_timeout = _testcapi.INT_MAX + 1 + else: + large_timeout = 2147483648 self.cli.connect((HOST, self.port)) try: self.cli.settimeout(large_timeout) From c4303763dac4494300e299e54c079a4a11931a55 Mon Sep 17 00:00:00 2001 From: mpage Date: Mon, 2 Dec 2024 10:13:30 -0800 Subject: [PATCH 077/427] gh-127411: Fix invalid conversion of load of TLBC array when compiled in C++ (#127466) Cast the result of the load to the correct type --- Include/internal/pycore_code.h | 9 ++++++++- Include/internal/pycore_frame.h | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index a0acf76db6f04d..d607a54aa4a2f5 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -609,12 +609,19 @@ PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; #ifdef Py_GIL_DISABLED +static inline _PyCodeArray * +_PyCode_GetTLBCArray(PyCodeObject *co) +{ + return _Py_STATIC_CAST(_PyCodeArray *, + _Py_atomic_load_ptr_acquire(&co->co_tlbc)); +} + // Return a pointer to the thread-local bytecode for the current thread, if it // exists. static inline _Py_CODEUNIT * _PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co) { - _PyCodeArray *code = _Py_atomic_load_ptr_acquire(&co->co_tlbc); + _PyCodeArray *code = _PyCode_GetTLBCArray(co); int32_t idx = ((_PyThreadStateImpl*) tstate)->tlbc_index; if (idx < code->size && code->entries[idx] != NULL) { return (_Py_CODEUNIT *) code->entries[idx]; diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index b786c5f49e9831..96ae4dd22ecb43 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -94,7 +94,7 @@ _PyFrame_GetBytecode(_PyInterpreterFrame *f) { #ifdef Py_GIL_DISABLED PyCodeObject *co = _PyFrame_GetCode(f); - _PyCodeArray *tlbc = _Py_atomic_load_ptr_acquire(&co->co_tlbc); + _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); return (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; #else From c7dec02de2ed4baf3cd22ad094350265b52c18af Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 2 Dec 2024 19:38:26 +0000 Subject: [PATCH 078/427] gh-127521: Mark list as "shared" before resizing if necessary (#127524) In the free threading build, if a non-owning thread resizes a list, it must use QSBR to free the old list array because there may be a concurrent access (without a lock) from the owning thread. To match the pattern in dictobject.c, we just mark the list as "shared" before resizing if it's from a non-owning thread and not already marked as shared. --- Objects/listobject.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Objects/listobject.c b/Objects/listobject.c index 4b24f4a428e18b..8abe9e8933420b 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -68,6 +68,20 @@ free_list_items(PyObject** items, bool use_qsbr) #endif } +static void +ensure_shared_on_resize(PyListObject *self) +{ +#ifdef Py_GIL_DISABLED + // Ensure that the list array is freed using QSBR if we are not the + // owning thread. + if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && + !_PyObject_GC_IS_SHARED(self)) + { + _PyObject_GC_SET_SHARED(self); + } +#endif +} + /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content * of the new slots at exit is undefined heap trash; it's the caller's @@ -117,6 +131,8 @@ list_resize(PyListObject *self, Py_ssize_t newsize) if (newsize == 0) new_allocated = 0; + ensure_shared_on_resize(self); + #ifdef Py_GIL_DISABLED _PyListArray *array = list_allocate_array(new_allocated); if (array == NULL) { @@ -804,6 +820,9 @@ list_clear_impl(PyListObject *a, bool is_resize) Py_XDECREF(items[i]); } #ifdef Py_GIL_DISABLED + if (is_resize) { + ensure_shared_on_resize(a); + } bool use_qsbr = is_resize && _PyObject_GC_IS_SHARED(a); #else bool use_qsbr = false; @@ -3069,6 +3088,7 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) Py_XDECREF(final_ob_item[i]); } #ifdef Py_GIL_DISABLED + ensure_shared_on_resize(self); bool use_qsbr = _PyObject_GC_IS_SHARED(self); #else bool use_qsbr = false; From edefb8678a11a20bdcdcbb8bb6a62ae22101bb51 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 2 Dec 2024 15:17:08 -0500 Subject: [PATCH 079/427] gh-127518: Fix pystats build after #127169 (#127526) gh-127518: Fix pystats build after #127619 --- Python/specialize.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index 504eef4f448429..ba13b02a29b133 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1816,8 +1816,9 @@ _Py_Specialize_BinarySubscr( #ifdef Py_STATS static int -store_subscr_fail_kind(PyObject *container_type) +store_subscr_fail_kind(PyObject *container, PyObject *sub) { + PyTypeObject *container_type = Py_TYPE(container); PyMappingMethods *as_mapping = container_type->tp_as_mapping; if (as_mapping && (as_mapping->mp_ass_subscript == PyDict_Type.tp_as_mapping->mp_ass_subscript)) { @@ -1915,7 +1916,7 @@ _Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_COD specialize(instr, STORE_SUBSCR_DICT); return; } - SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container_type)); + SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container, sub)); unspecialize(instr); } From bfb0788bfcaab7474c1be0605552744e15082ee9 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 3 Dec 2024 00:30:24 +0100 Subject: [PATCH 080/427] gh-127111: Emscripten Make web example work again (#127113) Moves the Emscripten web example into a standalone folder, and updates Makefile targets to build the web example. Instructions for usage have also been added. --- Makefile.pre.in | 51 +++++--- ...-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst | 2 + Tools/wasm/README.md | 122 ++++++++++++------ .../{ => emscripten/web_example}/python.html | 4 +- .../web_example/python.worker.mjs} | 25 +++- .../web_example/server.py} | 8 +- .../web_example}/wasm_assets.py | 25 ++-- configure | 14 +- configure.ac | 16 ++- 9 files changed, 175 insertions(+), 92 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst rename Tools/wasm/{ => emscripten/web_example}/python.html (99%) rename Tools/wasm/{python.worker.js => emscripten/web_example/python.worker.mjs} (65%) rename Tools/wasm/{wasm_webserver.py => emscripten/web_example/server.py} (85%) rename Tools/wasm/{ => emscripten/web_example}/wasm_assets.py (91%) diff --git a/Makefile.pre.in b/Makefile.pre.in index 724354746b8d81..dd8a3ab82eacd2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -269,10 +269,6 @@ SRCDIRS= @SRCDIRS@ # Other subdirectories SUBDIRSTOO= Include Lib Misc -# assets for Emscripten browser builds -WASM_ASSETS_DIR=.$(prefix) -WASM_STDLIB=$(WASM_ASSETS_DIR)/lib/python$(VERSION)/os.py - # Files and directories to be distributed CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in DISTFILES= README.rst ChangeLog $(CONFIGFILES) @@ -737,6 +733,9 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ python-config checksharedmods +.PHONY: build_emscripten +build_emscripten: build_wasm web_example + # Check that the source is clean when building out of source. .PHONY: check-clean-src check-clean-src: @@ -1016,23 +1015,38 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) else true; \ fi -# wasm32-emscripten browser build -# wasm assets directory is relative to current build dir, e.g. "./usr/local". -# --preload-file turns a relative asset path into an absolute path. +# wasm32-emscripten browser web example + +WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/ +web_example/python.html: $(WEBEX_DIR)/python.html + @mkdir -p web_example + @cp $< $@ + +web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs + @mkdir -p web_example + @cp $< $@ -.PHONY: wasm_stdlib -wasm_stdlib: $(WASM_STDLIB) -$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ - $(srcdir)/Tools/wasm/wasm_assets.py \ +web_example/server.py: $(WEBEX_DIR)/server.py + @mkdir -p web_example + @cp $< $@ + +WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip +$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ + $(WEBEX_DIR)/wasm_assets.py \ Makefile pybuilddir.txt Modules/Setup.local - $(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \ - --buildroot . --prefix $(prefix) + $(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \ + --buildroot . --prefix $(prefix) -o $@ -python.html: $(srcdir)/Tools/wasm/python.html python.worker.js - @cp $(srcdir)/Tools/wasm/python.html $@ +web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) + @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ + echo "Can only build web_example when target is Emscripten" ;\ + exit 1 ;\ + fi + cp python.mjs web_example/python.mjs + cp python.wasm web_example/python.wasm -python.worker.js: $(srcdir)/Tools/wasm/python.worker.js - @cp $(srcdir)/Tools/wasm/python.worker.js $@ +.PHONY: web_example +web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB) ############################################################################ # Header files @@ -3053,8 +3067,7 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f _bootstrap_python - -rm -f python.html python*.js python.data python*.symbols python*.map - -rm -f $(WASM_STDLIB) + -rm -rf web_example python.mjs python.wasm python*.symbols python*.map -rm -f Programs/_testembed Programs/_freeze_module -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h diff --git a/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst new file mode 100644 index 00000000000000..d90067cd3bfaa3 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst @@ -0,0 +1,2 @@ +Updated the Emscripten web example to use ES6 modules and be built into a +distinct ``web_example`` subfolder. diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 3f4211fb1dfb28..4802d9683de52e 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -23,9 +23,9 @@ https://github.com/psf/webassembly for more information. To cross compile to the ``wasm32-emscripten`` platform you need [the Emscripten compiler toolchain](https://emscripten.org/), -a Python interpreter, and an installation of Node version 18 or newer. Emscripten -version 3.1.42 or newer is recommended. All commands below are relative to a checkout -of the Python repository. +a Python interpreter, and an installation of Node version 18 or newer. +Emscripten version 3.1.73 or newer is recommended. All commands below are +relative to a checkout of the Python repository. #### Install [the Emscripten compiler toolchain](https://emscripten.org/docs/getting_started/downloads.html) @@ -50,7 +50,7 @@ sourced. Otherwise the source script removes the environment variable. export EM_COMPILER_WRAPPER=ccache ``` -### Compile and build Python interpreter +#### Compile and build Python interpreter You can use `python Tools/wasm/emscripten` to compile and build targetting Emscripten. You can do everything at once with: @@ -70,6 +70,88 @@ instance, to do a debug build, you can use: python Tools/wasm/emscripten build --with-py-debug ``` +### Running from node + +If you want to run the normal Python CLI, you can use `python.sh`. It takes the +same options as the normal Python CLI entrypoint, though the REPL does not +function and will crash. + +`python.sh` invokes `node_entry.mjs` which imports the Emscripten module for the +Python process and starts it up with the appropriate settings. If you wish to +make a node application that "embeds" the interpreter instead of acting like the +CLI you will need to write your own alternative to `node_entry.mjs`. + + +### The Web Example + +When building for Emscripten, the web example will be built automatically. It is +in the ``web_example`` directory. To run the web example, ``cd`` into the +``web_example`` directory, then run ``python server.py``. This will start a web +server; you can then visit ``http://localhost:8000/python.html`` in a browser to +see a simple REPL example. + +The web example relies on a bug fix in Emscripten version 3.1.73 so if you build +with earlier versions of Emscripten it may not work. The web example uses +``SharedArrayBuffer``. For security reasons browsers only provide +``SharedArrayBuffer`` in secure environments with cross-origin isolation. The +webserver must send cross-origin headers and correct MIME types for the +JavaScript and WebAssembly files. Otherwise the terminal will fail to load with +an error message like ``ReferenceError: SharedArrayBuffer is not defined``. See +more information here: +https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements + +Note that ``SharedArrayBuffer`` is _not required_ to use Python itself, only the +web example. If cross-origin isolation is not appropriate for your use case you +may make your own application embedding `python.mjs` which does not use +``SharedArrayBuffer`` and serve it without the cross-origin isolation headers. + +### Embedding Python in a custom JavaScript application + +You can look at `python.worker.mjs` and `node_entry.mjs` for inspiration. At a +minimum you must import ``createEmscriptenModule`` and you need to call +``createEmscriptenModule`` with an appropriate settings object. This settings +object will need a prerun hook that installs the Python standard library into +the Emscripten file system. + +#### NodeJs + +In Node, you can use the NodeFS to mount the standard library in your native +file system into the Emscripten file system: +```js +import createEmscriptenModule from "./python.mjs"; + +await createEmscriptenModule({ + preRun(Module) { + Module.FS.mount( + Module.FS.filesystems.NODEFS, + { root: "/path/to/python/stdlib" }, + "/lib/", + ); + }, +}); +``` + +#### Browser + +In the browser, the simplest approach is to put the standard library in a zip +file it and install it. With Python 3.14 this could look like: +```js +import createEmscriptenModule from "./python.mjs"; + +await createEmscriptenModule({ + async preRun(Module) { + Module.FS.mkdirTree("/lib/python3.14/lib-dynload/"); + Module.addRunDependency("install-stdlib"); + const resp = await fetch("python3.14.zip"); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { + canOwn: true, + }); + Module.removeRunDependency("install-stdlib"); + }, +}); +``` + ### Limitations and issues #### Network stack @@ -151,38 +233,6 @@ python Tools/wasm/emscripten build --with-py-debug - Test modules are disabled by default. Use ``--enable-test-modules`` build test modules like ``_testcapi``. -### wasm32-emscripten in node - -Node builds use ``NODERAWFS``. - -- Node RawFS allows direct access to the host file system without need to - perform ``FS.mount()`` call. - -### Hosting Python WASM builds - -The simple REPL terminal uses SharedArrayBuffer. For security reasons -browsers only provide the feature in secure environments with cross-origin -isolation. The webserver must send cross-origin headers and correct MIME types -for the JavaScript and WASM files. Otherwise the terminal will fail to load -with an error message like ``Browsers disable shared array buffer``. - -#### Apache HTTP .htaccess - -Place a ``.htaccess`` file in the same directory as ``python.wasm``. - -``` -# .htaccess -Header set Cross-Origin-Opener-Policy same-origin -Header set Cross-Origin-Embedder-Policy require-corp - -AddType application/javascript js -AddType application/wasm wasm - - - AddOutputFilterByType DEFLATE text/html application/javascript application/wasm - -``` - ## WASI (wasm32-wasi) See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi). diff --git a/Tools/wasm/python.html b/Tools/wasm/emscripten/web_example/python.html similarity index 99% rename from Tools/wasm/python.html rename to Tools/wasm/emscripten/web_example/python.html index 81a035a5c4cd93..fae1e9ad4e8acb 100644 --- a/Tools/wasm/python.html +++ b/Tools/wasm/emscripten/web_example/python.html @@ -47,7 +47,7 @@ async initialiseWorker() { if (!this.worker) { - this.worker = new Worker(this.workerURL) + this.worker = new Worker(this.workerURL, {type: "module"}) this.worker.addEventListener('message', this.handleMessageFromWorker) } } @@ -347,7 +347,7 @@ programRunning(false) } - const pythonWorkerManager = new WorkerManager('./python.worker.js', stdio, readyCallback, finishedCallback) + const pythonWorkerManager = new WorkerManager('./python.worker.mjs', stdio, readyCallback, finishedCallback) } diff --git a/Tools/wasm/python.worker.js b/Tools/wasm/emscripten/web_example/python.worker.mjs similarity index 65% rename from Tools/wasm/python.worker.js rename to Tools/wasm/emscripten/web_example/python.worker.mjs index 4ce4e16fc0fa19..42c2e1e08af24b 100644 --- a/Tools/wasm/python.worker.js +++ b/Tools/wasm/emscripten/web_example/python.worker.mjs @@ -1,3 +1,5 @@ +import createEmscriptenModule from "./python.mjs"; + class StdinBuffer { constructor() { this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT) @@ -59,24 +61,40 @@ const stderr = (charCode) => { const stdinBuffer = new StdinBuffer() -var Module = { +const emscriptenSettings = { noInitialRun: true, stdin: stdinBuffer.stdin, stdout: stdout, stderr: stderr, onRuntimeInitialized: () => { postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab}) + }, + async preRun(Module) { + const versionHex = Module.HEAPU32[Module._Py_Version/4].toString(16); + const versionTuple = versionHex.padStart(8, "0").match(/.{1,2}/g).map((x) => parseInt(x, 16)); + const [major, minor, ..._] = versionTuple; + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); + Module.addRunDependency("install-stdlib"); + const resp = await fetch(`python${major}.${minor}.zip`); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile(`/lib/python${major}${minor}.zip`, new Uint8Array(stdlibBuffer), { canOwn: true }); + Module.removeRunDependency("install-stdlib"); } } -onmessage = (event) => { +const modulePromise = createEmscriptenModule(emscriptenSettings); + + +onmessage = async (event) => { if (event.data.type === 'run') { + const Module = await modulePromise; if (event.data.files) { for (const [filename, contents] of Object.entries(event.data.files)) { Module.FS.writeFile(filename, contents) } } - const ret = callMain(event.data.args) + const ret = Module.callMain(event.data.args); postMessage({ type: 'finished', returnCode: ret @@ -84,4 +102,3 @@ onmessage = (event) => { } } -importScripts('python.js') diff --git a/Tools/wasm/wasm_webserver.py b/Tools/wasm/emscripten/web_example/server.py similarity index 85% rename from Tools/wasm/wasm_webserver.py rename to Tools/wasm/emscripten/web_example/server.py index 3d1d5d42a1e8c4..768e6f84e07798 100755 --- a/Tools/wasm/wasm_webserver.py +++ b/Tools/wasm/emscripten/web_example/server.py @@ -14,13 +14,6 @@ class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler): - extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy() - extensions_map.update( - { - ".wasm": "application/wasm", - } - ) - def end_headers(self) -> None: self.send_my_headers() super().end_headers() @@ -42,5 +35,6 @@ def main() -> None: bind=args.bind, ) + if __name__ == "__main__": main() diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/emscripten/web_example/wasm_assets.py similarity index 91% rename from Tools/wasm/wasm_assets.py rename to Tools/wasm/emscripten/web_example/wasm_assets.py index ffa5e303412c46..7f0fa7ae7c10ec 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/emscripten/web_example/wasm_assets.py @@ -19,7 +19,7 @@ from typing import Dict # source directory -SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() +SRCDIR = pathlib.Path(__file__).parents[4].absolute() SRCDIR_LIB = SRCDIR / "Lib" @@ -28,9 +28,7 @@ WASM_STDLIB_ZIP = ( WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" ) -WASM_STDLIB = ( - WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" -) +WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" @@ -114,9 +112,7 @@ def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path: assert isinstance(args.builddir, pathlib.Path) data_name: str = sysconfig._get_sysconfigdata_name() # type: ignore[attr-defined] if not data_name.startswith(SYSCONFIG_NAMES): - raise ValueError( - f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES - ) + raise ValueError(f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES) filename = data_name + ".py" return args.builddir / filename @@ -131,7 +127,7 @@ def filterfunc(filename: str) -> bool: return pathname not in args.omit_files_absolute with zipfile.PyZipFile( - args.wasm_stdlib_zip, + args.output, mode="w", compression=args.compression, optimize=optimize, @@ -195,6 +191,12 @@ def path(val: str) -> pathlib.Path: default=pathlib.Path("/usr/local"), type=path, ) +parser.add_argument( + "-o", + "--output", + help="output file", + type=path, +) def main() -> None: @@ -204,7 +206,6 @@ def main() -> None: args.srcdir = SRCDIR args.srcdir_lib = SRCDIR_LIB args.wasm_root = args.buildroot / relative_prefix - args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP args.wasm_stdlib = args.wasm_root / WASM_STDLIB args.wasm_dynload = args.wasm_root / WASM_DYNLOAD @@ -234,12 +235,10 @@ def main() -> None: args.wasm_dynload.mkdir(parents=True, exist_ok=True) marker = args.wasm_dynload / ".empty" marker.touch() - # os.py is a marker for finding the correct lib directory. - shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib) # The rest of stdlib that's useful in a WASM context. create_stdlib_zip(args) - size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2) - parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n") + size = round(args.output.stat().st_size / 1024**2, 2) + parser.exit(0, f"Created {args.output} ({size} MiB)\n") if __name__ == "__main__": diff --git a/configure b/configure index 4e4043260ed2df..7efda041ae69d4 100755 --- a/configure +++ b/configure @@ -8333,8 +8333,12 @@ fi fi -elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then - DEF_MAKE_ALL_RULE="build_wasm" +elif test "$ac_sys_system" = "Emscripten"; then + DEF_MAKE_ALL_RULE="build_emscripten" + REQUIRE_PGO="no" + DEF_MAKE_RULE="all" +elif test "$ac_sys_system" = "WASI"; then + DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" DEF_MAKE_RULE="all" else @@ -9425,12 +9429,13 @@ else $as_nop wasm_debug=no fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520" + as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520" as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS" + as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,callMain" + as_fn_append LDFLAGS_NODIST " -sEXPORTED_FUNCTIONS=_main,_Py_Version" if test "x$enable_wasm_dynamic_linking" = xyes then : @@ -9447,7 +9452,6 @@ then : as_fn_append LINKFORSHARED " -sPROXY_TO_PTHREAD" fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH" as_fn_append LDFLAGS_NODIST " -sEXIT_RUNTIME" WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" diff --git a/configure.ac b/configure.ac index 4cfced10432491..15f7d07f22473b 100644 --- a/configure.ac +++ b/configure.ac @@ -1854,9 +1854,13 @@ if test "$Py_OPT" = 'true' ; then LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition" ], [], [-Werror]) ]) -elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then - dnl Emscripten does not support shared extensions yet. Build - dnl "python.[js,wasm]", "pybuilddir.txt", and "platform" files. +elif test "$ac_sys_system" = "Emscripten"; then + dnl Build "python.[js,wasm]", "pybuilddir.txt", and "platform" files. + DEF_MAKE_ALL_RULE="build_emscripten" + REQUIRE_PGO="no" + DEF_MAKE_RULE="all" +elif test "$ac_sys_system" = "WASI"; then + dnl Build "python.wasm", "pybuilddir.txt", and "platform" files. DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" DEF_MAKE_RULE="all" @@ -2321,14 +2325,15 @@ AS_CASE([$ac_sys_system], AS_VAR_IF([Py_DEBUG], [yes], [wasm_debug=yes], [wasm_debug=no]) dnl Start with 20 MB and allow to grow - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"]) dnl map int64_t and uint64_t to JS bigint AS_VAR_APPEND([LDFLAGS_NODIST], [" -sWASM_BIGINT"]) dnl Include file system support AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) @@ -2339,7 +2344,6 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([LDFLAGS_NODIST], [" -sUSE_PTHREADS"]) AS_VAR_APPEND([LINKFORSHARED], [" -sPROXY_TO_PTHREAD"]) ]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH"]) dnl not completely sure whether or not we want -sEXIT_RUNTIME, keeping it for now. AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXIT_RUNTIME"]) WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" From dffb90911a585a0921664c8b1c229d0883e65ee7 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 2 Dec 2024 20:45:36 -0600 Subject: [PATCH 081/427] Speed-up lazy heapq import in collections (gh-127538) --- Lib/collections/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index d688141f9b183d..78229ac54b80da 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -59,6 +59,8 @@ except ImportError: pass +heapq = None # Lazily imported + ################################################################################ ### OrderedDict @@ -633,7 +635,10 @@ def most_common(self, n=None): return sorted(self.items(), key=_itemgetter(1), reverse=True) # Lazy import to speedup Python startup time - import heapq + global heapq + if heapq is None: + import heapq + return heapq.nlargest(n, self.items(), key=_itemgetter(1)) def elements(self): From 8c3fd1f245fbdc747966daedfd22ed48491309dc Mon Sep 17 00:00:00 2001 From: Alexander Stepchenko Date: Tue, 3 Dec 2024 09:52:12 +0300 Subject: [PATCH 082/427] docs(logging): fix phrasing from "operation on" to "operate on" (#127543) --- Doc/howto/logging.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 3182d5664ab6ec..2982cf88bf97b4 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -127,7 +127,7 @@ that; formatting options will also be explained later. Notice that in this example, we use functions directly on the ``logging`` module, like ``logging.debug``, rather than creating a logger and calling -functions on it. These functions operation on the root logger, but can be useful +functions on it. These functions operate on the root logger, but can be useful as they will call :func:`~logging.basicConfig` for you if it has not been called yet, like in this example. In larger programs you'll usually want to control the logging configuration explicitly however - so for that reason as well as others, it's From 35d37d6592d1be71ea76042165f6cbfa6c4c3a17 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 3 Dec 2024 13:30:27 +0100 Subject: [PATCH 083/427] gh-127253: Note that Stable ABI is about ABI stability (GH-127254) --- Doc/c-api/stable.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 5b9e43874c7f2b..124e58cf950b7a 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -66,7 +66,7 @@ Limited C API Python 3.2 introduced the *Limited API*, a subset of Python's C API. Extensions that only use the Limited API can be -compiled once and work with multiple versions of Python. +compiled once and be loaded on multiple versions of Python. Contents of the Limited API are :ref:`listed below `. .. c:macro:: Py_LIMITED_API @@ -76,7 +76,7 @@ Contents of the Limited API are :ref:`listed below `. Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` corresponding to the lowest Python version your extension supports. - The extension will work without recompilation with all Python 3 releases + The extension will be ABI-compatible with all Python 3 releases from the specified one onward, and can use Limited API introduced up to that version. @@ -94,7 +94,15 @@ Stable ABI ---------- To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain compatible across Python 3.x versions. +remain ABI-compatible across Python 3.x versions. + +.. note:: + + The Stable ABI prevents ABI issues, like linker errors due to missing + symbols or data corruption due to changes in structure layouts or function + signatures. + However, other changes in Python can change the *behavior* of extensions. + See Python's Backwards Compatibility Policy (:pep:`387`) for details. The Stable ABI contains symbols exposed in the :ref:`Limited API `, but also other ones – for example, functions necessary to From 84ff1313d04e8cbeec7b2cbe4503d86f1f5b449d Mon Sep 17 00:00:00 2001 From: "RUANG (James Roy)" Date: Tue, 3 Dec 2024 20:45:50 +0800 Subject: [PATCH 084/427] gh-126585: Add EHWPOISON error code (#126586) --- Doc/library/errno.rst | 7 +++++++ Doc/whatsnew/3.14.rst | 5 +++++ .../Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst | 1 + Modules/errnomodule.c | 3 +++ 4 files changed, 16 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 4983b8961b1c3f..824d489818fac9 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -613,6 +613,13 @@ defined by the module. The specific list of defined symbols is available as No route to host +.. data:: EHWPOISON + + Memory page has hardware error. + + .. versionadded:: next + + .. data:: EALREADY Operation already in progress. This error is mapped to the diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 75d027d33ccd16..7bb9657e6ed9da 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -345,6 +345,11 @@ dis This feature is also exposed via :option:`dis --show-positions`. (Contributed by Bénédikt Tran in :gh:`123165`.) +errno +----- + +* Add :data:`errno.EHWPOISON` error code. + (Contributed by James Roy in :gh:`126585`.) fractions --------- diff --git a/Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst b/Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst new file mode 100644 index 00000000000000..ae1392e2caf387 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst @@ -0,0 +1 @@ +Add :data:`~errno.EHWPOISON` error code to :mod:`errno`. diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 3f96f2f846d612..9557d68e759497 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -845,6 +845,9 @@ errno_exec(PyObject *module) #ifdef ENOKEY add_errcode("ENOKEY", ENOKEY, "Required key not available"); #endif +#ifdef EHWPOISON + add_errcode("EHWPOISON", EHWPOISON, "Memory page has hardware error"); +#endif #ifdef EKEYEXPIRED add_errcode("EKEYEXPIRED", EKEYEXPIRED, "Key has expired"); #endif From 979bf2489d0c59ae451b97d7e3c148f47e259f0b Mon Sep 17 00:00:00 2001 From: Daniele Parmeggiani <8658291+dpdani@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:41:53 +0100 Subject: [PATCH 085/427] gh-117657: TSAN Fix races in `PyMember_Get` and `PyMember_Set` for C extensions (GH-123211) --- Include/cpython/pyatomic.h | 45 ++++ Include/cpython/pyatomic_gcc.h | 64 +++++ Include/cpython/pyatomic_msc.h | 102 ++++++++ Include/cpython/pyatomic_std.h | 136 ++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 73 +++++- Lib/test/libregrtest/tsan.py | 1 + Lib/test/test_capi/test_structmembers.py | 13 + Lib/test/test_free_threading/test_slots.py | 244 ++++++++++++++++++ Modules/_testcapi/structmember.c | 10 +- Python/structmember.c | 97 +++---- 10 files changed, 735 insertions(+), 50 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 4ecef4f56edf42..6d106c1b499c69 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -321,6 +321,27 @@ _Py_atomic_load_ptr(const void *obj); static inline int _Py_atomic_load_int_relaxed(const int *obj); +static inline char +_Py_atomic_load_char_relaxed(const char *obj); + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj); + +static inline short +_Py_atomic_load_short_relaxed(const short *obj); + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj); + +static inline long +_Py_atomic_load_long_relaxed(const long *obj); + +static inline double +_Py_atomic_load_double_relaxed(const double *obj); + +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -458,6 +479,30 @@ static inline void _Py_atomic_store_ullong_relaxed(unsigned long long *obj, unsigned long long value); +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value); + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value); + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value); + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value); + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value); + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value); + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value); + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value); + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index ef09954d53ac1d..b179e4c9a185a9 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -306,6 +306,34 @@ static inline int _Py_atomic_load_int_relaxed(const int *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ float ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ double ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -362,6 +390,10 @@ static inline unsigned long long _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + // --- _Py_atomic_store ------------------------------------------------------ @@ -485,6 +517,38 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, unsigned long long value) { __atomic_store_n(obj, value, __ATOMIC_RELAXED); } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 84da21bdcbff4f..d155955df0cddf 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -634,6 +634,48 @@ _Py_atomic_load_int_relaxed(const int *obj) return *(volatile int *)obj; } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ + return *(volatile char *)obj; +} + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + return *(volatile unsigned char *)obj; +} + +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + return *(volatile short *)obj; +} + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + return *(volatile unsigned short *)obj; +} + +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + return *(volatile long *)obj; +} + +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + return *(volatile float *)obj; +} + +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + return *(volatile double *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -718,6 +760,12 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) return *(volatile unsigned long long *)obj; } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + return *(volatile long long *)obj; +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -899,6 +947,60 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, *(volatile unsigned long long *)obj = value; } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + *(volatile char *)obj = value; +} + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ + *(volatile unsigned char *)obj = value; +} + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value) +{ + *(volatile short *)obj = value; +} + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ + *(volatile unsigned short *)obj = value; +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + *(volatile unsigned int *)obj = value; +} + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value) +{ + *(volatile long *)obj = value; +} + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value) +{ + *(volatile float *)obj = value; +} + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value) +{ + *(volatile double *)obj = value; +} + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ + *(volatile long long *)obj = value; +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 7c71e94c68f8e6..69a8b9e615ea5f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -515,6 +515,62 @@ _Py_atomic_load_int_relaxed(const int *obj) memory_order_relaxed); } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(char)*)obj, + memory_order_relaxed); +} + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned char)*)obj, + memory_order_relaxed); +} + +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(short)*)obj, + memory_order_relaxed); +} + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned short)*)obj, + memory_order_relaxed); +} + +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long)*)obj, + memory_order_relaxed); +} + +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(float)*)obj, + memory_order_relaxed); +} + +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(double)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -627,6 +683,14 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) memory_order_relaxed); } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long long)*)obj, + memory_order_relaxed); +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -852,6 +916,78 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, memory_order_relaxed); } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(char)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned char)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(short)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned short)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(long)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(float)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(double)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(long long)*)obj, value, + memory_order_relaxed); +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index a1bb383bcd22e9..d755d03a5fa190 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -61,6 +61,54 @@ extern "C" { _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ + _Py_atomic_store_char_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ + _Py_atomic_load_char_relaxed(&value) +#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) \ + _Py_atomic_store_uchar_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) \ + _Py_atomic_load_uchar_relaxed(&value) +#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) \ + _Py_atomic_store_short_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) \ + _Py_atomic_load_short_relaxed(&value) +#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) \ + _Py_atomic_store_ushort_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ + _Py_atomic_load_ushort_relaxed(&value) +#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \ + _Py_atomic_store_int_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_INT_RELAXED(value) \ + _Py_atomic_load_int_relaxed(&value) +#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) \ + _Py_atomic_store_uint_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ + _Py_atomic_load_uint_relaxed(&value) +#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) \ + _Py_atomic_store_long_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_LONG_RELAXED(value) \ + _Py_atomic_load_long_relaxed(&value) +#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) \ + _Py_atomic_store_ulong_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ + _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) \ + _Py_atomic_store_float_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ + _Py_atomic_load_float_relaxed(&value) +#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) \ + _Py_atomic_store_double_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) \ + _Py_atomic_load_double_relaxed(&value) +#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) \ + _Py_atomic_store_llong_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) \ + _Py_atomic_load_llong_relaxed(&value) +#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) \ + _Py_atomic_store_ullong_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ + _Py_atomic_load_ullong_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -68,7 +116,6 @@ extern "C" { #define FT_ATOMIC_LOAD_SSIZE(value) value #define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value -#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value @@ -85,6 +132,30 @@ extern "C" { #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value +#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value +#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) value +#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value +#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT_RELAXED(value) value +#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT_RELAXED(value) value +#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_LONG_RELAXED(value) value +#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) value +#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) value +#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) value +#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value +#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #endif diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 822ac0f4044d9e..00d5779d950e72 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -25,6 +25,7 @@ 'test_threading_local', 'test_threadsignals', 'test_weakref', + 'test_free_threading.test_slots', ] diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index 6b27dc512a7d15..ae9168fc39243f 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -37,6 +37,9 @@ def _make_test_object(cls): 9.99999,# T_FLOAT 10.1010101010, # T_DOUBLE "hi", # T_STRING_INPLACE + 12, # T_LONGLONG + 13, # T_ULONGLONG + "c", # T_CHAR ) @@ -162,6 +165,16 @@ def test_inplace_string(self): self.assertRaises(TypeError, setattr, ts, "T_STRING_INPLACE", "s") self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + def test_char(self): + ts = self.ts + self.assertEqual(ts.T_CHAR, "c") + ts.T_CHAR = "z" + self.assertEqual(ts.T_CHAR, "z") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", "") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", b"a") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", bytearray(b"b")) + self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): cls = _test_structmembersType_OldAPI diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py index 758f74f54d0b56..a3b9f4b0175ae7 100644 --- a/Lib/test/test_free_threading/test_slots.py +++ b/Lib/test/test_free_threading/test_slots.py @@ -1,3 +1,4 @@ +import _testcapi import threading from test.support import threading_helper from unittest import TestCase @@ -41,3 +42,246 @@ def reader(): assert 0 <= eggs <= iters run_in_threads([writer, reader, reader, reader]) + + def test_T_BOOL(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + # different code paths for True and False + spam_old.T_BOOL = True + spam_new.T_BOOL = True + spam_old.T_BOOL = False + spam_new.T_BOOL = False + + def reader(): + for _ in range(1_000): + spam_old.T_BOOL + spam_new.T_BOOL + + run_in_threads([writer, reader]) + + def test_T_BYTE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_BYTE = 0 + spam_new.T_BYTE = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_BYTE + spam_new.T_BYTE + + run_in_threads([writer, reader]) + + def test_T_UBYTE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_UBYTE = 0 + spam_new.T_UBYTE = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_UBYTE + spam_new.T_UBYTE + + run_in_threads([writer, reader]) + + def test_T_SHORT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_SHORT = 0 + spam_new.T_SHORT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_SHORT + spam_new.T_SHORT + + run_in_threads([writer, reader]) + + def test_T_USHORT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_USHORT = 0 + spam_new.T_USHORT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_USHORT + spam_new.T_USHORT + + run_in_threads([writer, reader]) + + def test_T_INT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_INT = 0 + spam_new.T_INT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_INT + spam_new.T_INT + + run_in_threads([writer, reader]) + + def test_T_UINT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_UINT = 0 + spam_new.T_UINT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_UINT + spam_new.T_UINT + + run_in_threads([writer, reader]) + + def test_T_LONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_LONG = 0 + spam_new.T_LONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_LONG + spam_new.T_LONG + + run_in_threads([writer, reader]) + + def test_T_ULONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_ULONG = 0 + spam_new.T_ULONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_ULONG + spam_new.T_ULONG + + run_in_threads([writer, reader]) + + def test_T_PYSSIZET(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_PYSSIZET = 0 + spam_new.T_PYSSIZET = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_PYSSIZET + spam_new.T_PYSSIZET + + run_in_threads([writer, reader]) + + def test_T_FLOAT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_FLOAT = 0.0 + spam_new.T_FLOAT = 0.0 + + def reader(): + for _ in range(1_000): + spam_old.T_FLOAT + spam_new.T_FLOAT + + run_in_threads([writer, reader]) + + def test_T_DOUBLE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_DOUBLE = 0.0 + spam_new.T_DOUBLE = 0.0 + + def reader(): + for _ in range(1_000): + spam_old.T_DOUBLE + spam_new.T_DOUBLE + + run_in_threads([writer, reader]) + + def test_T_LONGLONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_LONGLONG = 0 + spam_new.T_LONGLONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_LONGLONG + spam_new.T_LONGLONG + + run_in_threads([writer, reader]) + + def test_T_ULONGLONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_ULONGLONG = 0 + spam_new.T_ULONGLONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_ULONGLONG + spam_new.T_ULONGLONG + + run_in_threads([writer, reader]) + + def test_T_CHAR(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_CHAR = "c" + spam_new.T_CHAR = "c" + + def reader(): + for _ in range(1_000): + spam_old.T_CHAR + spam_new.T_CHAR + + run_in_threads([writer, reader]) diff --git a/Modules/_testcapi/structmember.c b/Modules/_testcapi/structmember.c index 096eaecd40855f..c1861db18c4af2 100644 --- a/Modules/_testcapi/structmember.c +++ b/Modules/_testcapi/structmember.c @@ -22,6 +22,7 @@ typedef struct { char inplace_member[6]; long long longlong_member; unsigned long long ulonglong_member; + char char_member; } all_structmembers; typedef struct { @@ -46,6 +47,7 @@ static struct PyMemberDef test_members_newapi[] = { {"T_STRING_INPLACE", Py_T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", Py_T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", Py_T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, + {"T_CHAR", Py_T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; @@ -56,9 +58,9 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "T_BOOL", "T_BYTE", "T_UBYTE", "T_SHORT", "T_USHORT", "T_INT", "T_UINT", "T_LONG", "T_ULONG", "T_PYSSIZET", "T_FLOAT", "T_DOUBLE", "T_STRING_INPLACE", - "T_LONGLONG", "T_ULONGLONG", + "T_LONGLONG", "T_ULONGLONG", "T_CHAR", NULL}; - static const char fmt[] = "|bbBhHiIlknfds#LK"; + static const char fmt[] = "|bbBhHiIlknfds#LKC"; test_structmembers *ob; const char *s = NULL; Py_ssize_t string_len = 0; @@ -82,7 +84,8 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) &ob->structmembers.double_member, &s, &string_len, &ob->structmembers.longlong_member, - &ob->structmembers.ulonglong_member)) + &ob->structmembers.ulonglong_member, + &ob->structmembers.char_member)) { Py_DECREF(ob); return NULL; @@ -132,6 +135,7 @@ static struct PyMemberDef test_members[] = { {"T_STRING_INPLACE", T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, + {"T_CHAR", T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; diff --git a/Python/structmember.c b/Python/structmember.c index d5e7ab83093dc8..d36e049d6b5d20 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -34,40 +34,40 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) const char* addr = obj_addr + l->offset; switch (l->type) { case Py_T_BOOL: - v = PyBool_FromLong(*(char*)addr); + v = PyBool_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_BYTE: - v = PyLong_FromLong(*(char*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_UBYTE: - v = PyLong_FromUnsignedLong(*(unsigned char*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UCHAR_RELAXED(*(unsigned char*)addr)); break; case Py_T_SHORT: - v = PyLong_FromLong(*(short*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_SHORT_RELAXED(*(short*)addr)); break; case Py_T_USHORT: - v = PyLong_FromUnsignedLong(*(unsigned short*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_USHORT_RELAXED(*(unsigned short*)addr)); break; case Py_T_INT: - v = PyLong_FromLong(*(int*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_INT_RELAXED(*(int*)addr)); break; case Py_T_UINT: - v = PyLong_FromUnsignedLong(*(unsigned int*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UINT_RELAXED(*(unsigned int*)addr)); break; case Py_T_LONG: - v = PyLong_FromLong(*(long*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_LONG_RELAXED(*(long*)addr)); break; case Py_T_ULONG: - v = PyLong_FromUnsignedLong(*(unsigned long*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_ULONG_RELAXED(*(unsigned long*)addr)); break; case Py_T_PYSSIZET: - v = PyLong_FromSsize_t(*(Py_ssize_t*)addr); + v = PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(*(Py_ssize_t*)addr)); break; case Py_T_FLOAT: - v = PyFloat_FromDouble((double)*(float*)addr); + v = PyFloat_FromDouble((double)FT_ATOMIC_LOAD_FLOAT_RELAXED(*(float*)addr)); break; case Py_T_DOUBLE: - v = PyFloat_FromDouble(*(double*)addr); + v = PyFloat_FromDouble(FT_ATOMIC_LOAD_DOUBLE_RELAXED(*(double*)addr)); break; case Py_T_STRING: if (*(char**)addr == NULL) { @@ -79,9 +79,11 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) case Py_T_STRING_INPLACE: v = PyUnicode_FromString((char*)addr); break; - case Py_T_CHAR: - v = PyUnicode_FromStringAndSize((char*)addr, 1); + case Py_T_CHAR: { + char char_val = FT_ATOMIC_LOAD_CHAR_RELAXED(*addr); + v = PyUnicode_FromStringAndSize(&char_val, 1); break; + } case _Py_T_OBJECT: v = *(PyObject **)addr; if (v == NULL) @@ -104,10 +106,10 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) #endif break; case Py_T_LONGLONG: - v = PyLong_FromLongLong(*(long long *)addr); + v = PyLong_FromLongLong(FT_ATOMIC_LOAD_LLONG_RELAXED(*(long long *)addr)); break; case Py_T_ULONGLONG: - v = PyLong_FromUnsignedLongLong(*(unsigned long long *)addr); + v = PyLong_FromUnsignedLongLong(FT_ATOMIC_LOAD_ULLONG_RELAXED(*(unsigned long long *)addr)); break; case _Py_T_NONE: // doesn't require free-threading code path @@ -169,16 +171,16 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; } if (v == Py_True) - *(char*)addr = (char) 1; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, 1); else - *(char*)addr = (char) 0; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, 0); break; } case Py_T_BYTE:{ long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(char*)addr = (char)long_val; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, (char)long_val); /* XXX: For compatibility, only warn about truncations for now. */ if ((long_val > CHAR_MAX) || (long_val < CHAR_MIN)) @@ -189,7 +191,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(unsigned char*)addr = (unsigned char)long_val; + FT_ATOMIC_STORE_UCHAR_RELAXED(*(unsigned char*)addr, (unsigned char)long_val); if ((long_val > UCHAR_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned char"); break; @@ -198,7 +200,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(short*)addr = (short)long_val; + FT_ATOMIC_STORE_SHORT_RELAXED(*(short*)addr, (short)long_val); if ((long_val > SHRT_MAX) || (long_val < SHRT_MIN)) WARN("Truncation of value to short"); break; @@ -207,7 +209,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(unsigned short*)addr = (unsigned short)long_val; + FT_ATOMIC_STORE_USHORT_RELAXED(*(unsigned short*)addr, (unsigned short)long_val); if ((long_val > USHRT_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned short"); break; @@ -216,7 +218,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(int *)addr = (int)long_val; + FT_ATOMIC_STORE_INT_RELAXED(*(int *)addr, (int)long_val); if ((long_val > INT_MAX) || (long_val < INT_MIN)) WARN("Truncation of value to int"); break; @@ -234,7 +236,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned int *)addr = (unsigned int)(unsigned long)long_val; + FT_ATOMIC_STORE_UINT_RELAXED(*(unsigned int *)addr, (unsigned int)(unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -243,19 +245,20 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned int*)addr = (unsigned int)ulong_val; + FT_ATOMIC_STORE_UINT_RELAXED(*(unsigned int *)addr, (unsigned int)ulong_val); if (ulong_val > UINT_MAX) { WARN("Truncation of value to unsigned int"); } } break; } - case Py_T_LONG:{ - *(long*)addr = PyLong_AsLong(v); - if ((*(long*)addr == -1) && PyErr_Occurred()) + case Py_T_LONG: { + const long long_val = PyLong_AsLong(v); + if ((long_val == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_LONG_RELAXED(*(long*)addr, long_val); break; - } + } case Py_T_ULONG: { /* XXX: For compatibility, accept negative int values as well. */ @@ -269,7 +272,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned long *)addr = (unsigned long)long_val; + FT_ATOMIC_STORE_ULONG_RELAXED(*(unsigned long *)addr, (unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -278,29 +281,31 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned long*)addr = ulong_val; + FT_ATOMIC_STORE_ULONG_RELAXED(*(unsigned long *)addr, ulong_val); } break; } - case Py_T_PYSSIZET:{ - *(Py_ssize_t*)addr = PyLong_AsSsize_t(v); - if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) - && PyErr_Occurred()) - return -1; + case Py_T_PYSSIZET: { + const Py_ssize_t ssize_val = PyLong_AsSsize_t(v); + if ((ssize_val == (Py_ssize_t)-1) && PyErr_Occurred()) + return -1; + FT_ATOMIC_STORE_SSIZE_RELAXED(*(Py_ssize_t*)addr, ssize_val); break; - } + } case Py_T_FLOAT:{ double double_val = PyFloat_AsDouble(v); if ((double_val == -1) && PyErr_Occurred()) return -1; - *(float*)addr = (float)double_val; + FT_ATOMIC_STORE_FLOAT_RELAXED(*(float*)addr, (float)double_val); break; } - case Py_T_DOUBLE: - *(double*)addr = PyFloat_AsDouble(v); - if ((*(double*)addr == -1) && PyErr_Occurred()) + case Py_T_DOUBLE: { + const double double_val = PyFloat_AsDouble(v); + if ((double_val == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_DOUBLE_RELAXED(*(double *) addr, double_val); break; + } case _Py_T_OBJECT: case Py_T_OBJECT_EX: Py_BEGIN_CRITICAL_SECTION(obj); @@ -318,7 +323,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_BadArgument(); return -1; } - *(char*)addr = string[0]; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, string[0]); break; } case Py_T_STRING: @@ -326,10 +331,10 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_SetString(PyExc_TypeError, "readonly attribute"); return -1; case Py_T_LONGLONG:{ - long long value; - *(long long*)addr = value = PyLong_AsLongLong(v); + long long value = PyLong_AsLongLong(v); if ((value == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_LLONG_RELAXED(*(long long*)addr, value); break; } case Py_T_ULONGLONG: { @@ -343,7 +348,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned long long *)addr = (unsigned long long)(long long)long_val; + FT_ATOMIC_STORE_ULLONG_RELAXED(*(unsigned long long *)addr, (unsigned long long)(long long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -352,7 +357,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulonglong_val == (unsigned long long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned long long*)addr = ulonglong_val; + FT_ATOMIC_STORE_ULLONG_RELAXED(*(unsigned long long *)addr, ulonglong_val); } break; } From 412e11fe6e37f15971ef855f88b8b01bb3297679 Mon Sep 17 00:00:00 2001 From: Jun Komoda <45822440+junkmd@users.noreply.github.com> Date: Wed, 4 Dec 2024 00:35:08 +0900 Subject: [PATCH 086/427] gh-127255: Make `CopyComPointer` public and add to `ctypes` doc. (GH-127275) --- Doc/library/ctypes.rst | 18 ++++++++++++++++++ Doc/whatsnew/3.14.rst | 5 ++++- Lib/ctypes/__init__.py | 2 +- .../test_ctypes/test_win32_com_foreign_func.py | 3 +-- ...4-11-25-15-02-44.gh-issue-127255.UXeljc.rst | 2 ++ 5 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index bd9529db9ee65a..bd88fa377fb39d 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1949,6 +1949,24 @@ Utility functions It behaves similar to ``pointer(obj)``, but the construction is a lot faster. +.. function:: CopyComPointer(src, dst) + + Copies a COM pointer from *src* to *dst* and returns the Windows specific + :c:type:`!HRESULT` value. + + If *src* is not ``NULL``, its ``AddRef`` method is called, incrementing the + reference count. + + In contrast, the reference count of *dst* will not be decremented before + assigning the new value. Unless *dst* is ``NULL``, the caller is responsible + for decrementing the reference count by calling its ``Release`` method when + necessary. + + .. availability:: Windows + + .. versionadded:: next + + .. function:: cast(obj, type) This function is similar to the cast operator in C. It returns a new instance diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 7bb9657e6ed9da..52a6d6e4340194 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -313,9 +313,12 @@ ctypes to help match a non-default ABI. (Contributed by Petr Viktorin in :gh:`97702`.) -* The :exc:`~ctypes.COMError` exception is now public. +* On Windows, the :exc:`~ctypes.COMError` exception is now public. (Contributed by Jun Komoda in :gh:`126686`.) +* On Windows, the :func:`~ctypes.CopyComPointer` function is now public. + (Contributed by Jun Komoda in :gh:`127275`.) + datetime -------- diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index ac6493892068e9..2f2b0ca9f38633 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -19,7 +19,7 @@ raise Exception("Version number mismatch", __version__, _ctypes_version) if _os.name == "nt": - from _ctypes import COMError, FormatError + from _ctypes import COMError, CopyComPointer, FormatError DEFAULT_MODE = RTLD_LOCAL if _os.name == "posix" and _sys.platform == "darwin": diff --git a/Lib/test/test_ctypes/test_win32_com_foreign_func.py b/Lib/test/test_ctypes/test_win32_com_foreign_func.py index 8d217fc17efa02..7e54f8f6c31d33 100644 --- a/Lib/test/test_ctypes/test_win32_com_foreign_func.py +++ b/Lib/test/test_ctypes/test_win32_com_foreign_func.py @@ -9,8 +9,7 @@ raise unittest.SkipTest("Windows-specific test") -from _ctypes import COMError, CopyComPointer -from ctypes import HRESULT +from ctypes import COMError, CopyComPointer, HRESULT COINIT_APARTMENTTHREADED = 0x2 diff --git a/Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst b/Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst new file mode 100644 index 00000000000000..9fe7815e93cf4f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst @@ -0,0 +1,2 @@ +The :func:`~ctypes.CopyComPointer` function is now public. +Previously, this was private and only available in ``_ctypes``. From 8ba9f5bca9c0ce6130e1f4ba761a68f74f8457d0 Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Tue, 3 Dec 2024 17:08:39 +0100 Subject: [PATCH 087/427] gh-127347: Document `traceback.print_list` (#127348) Previously, `traceback.print_list` didn't have a documentation entry and was not exposed in `traceback.__all__`. Now it has a documentation entry and is exposed in `__all__`. --- Doc/library/traceback.rst | 7 +++++++ Lib/test/test_traceback.py | 3 +-- Lib/traceback.py | 2 +- .../2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 100a92b73d5497..8f94fc448f2482 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -157,6 +157,13 @@ Module-Level Functions arguments have the same meaning as for :func:`print_stack`. +.. function:: print_list(extracted_list, file=None) + + Print the list of tuples as returned by :func:`extract_tb` or + :func:`extract_stack` as a formatted stack trace to the given file. + If *file* is ``None``, the output is written to :data:`sys.stderr`. + + .. function:: format_list(extracted_list) Given a list of tuples or :class:`FrameSummary` objects as returned by diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index ec69412f5511eb..ea8d9f2137aca5 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -4488,9 +4488,8 @@ class MiscTest(unittest.TestCase): def test_all(self): expected = set() - denylist = {'print_list'} for name in dir(traceback): - if name.startswith('_') or name in denylist: + if name.startswith('_'): continue module_object = getattr(traceback, name) if getattr(module_object, '__module__', None) == 'traceback': diff --git a/Lib/traceback.py b/Lib/traceback.py index f73149271b9bc9..6367c00e4d4b86 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -15,7 +15,7 @@ 'format_tb', 'print_exc', 'format_exc', 'print_exception', 'print_last', 'print_stack', 'print_tb', 'clear_frames', 'FrameSummary', 'StackSummary', 'TracebackException', - 'walk_stack', 'walk_tb'] + 'walk_stack', 'walk_tb', 'print_list'] # # Formatting and printing lists of traceback lines. diff --git a/Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst b/Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst new file mode 100644 index 00000000000000..79b3faa3d0d385 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst @@ -0,0 +1 @@ +Publicly expose :func:`traceback.print_list` in :attr:`!traceback.__all__`. From 0cb52220790d8bc70ec325fd89d52b5f3b7ad29c Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Tue, 3 Dec 2024 09:32:26 -0800 Subject: [PATCH 088/427] gh-115999: Specialize `LOAD_SUPER_ATTR` in free-threaded builds (gh-127128) Use existing helpers to atomically modify the bytecode. Add unit tests to ensure specializing is happening as expected. Add test_specialize.py that can be used with ThreadSanitizer to detect data races. Fix thread safety issue with cell_set_contents(). --- Lib/test/test_opcache.py | 39 ++++++++++++++++++++++++++++++++++++++ Objects/cellobject.c | 5 +++-- Python/bytecodes.c | 4 ++-- Python/ceval.c | 1 - Python/generated_cases.c.h | 4 ++-- Python/specialize.c | 19 +++++-------------- 6 files changed, 51 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index b989b21cd9b3a9..527114445ac13b 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1249,6 +1249,45 @@ def binary_op_add_unicode(): self.assert_specialized(binary_op_add_unicode, "BINARY_OP_ADD_UNICODE") self.assert_no_opcode(binary_op_add_unicode, "BINARY_OP") + @cpython_only + @requires_specialization_ft + def test_load_super_attr(self): + """Ensure that LOAD_SUPER_ATTR is specialized as expected.""" + + class A: + def __init__(self): + meth = super().__init__ + super().__init__() + + for _ in range(100): + A() + + self.assert_specialized(A.__init__, "LOAD_SUPER_ATTR_ATTR") + self.assert_specialized(A.__init__, "LOAD_SUPER_ATTR_METHOD") + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR") + + # Temporarily replace super() with something else. + real_super = super + + def fake_super(): + def init(self): + pass + + return init + + # Force unspecialize + globals()['super'] = fake_super + try: + # Should be unspecialized after enough calls. + for _ in range(100): + A() + finally: + globals()['super'] = real_super + + # Ensure the specialized instructions are not present + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR_ATTR") + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR_METHOD") + @cpython_only @requires_specialization_ft def test_contain_op(self): diff --git a/Objects/cellobject.c b/Objects/cellobject.c index 590c8a80857699..4ab9083af5e300 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -145,8 +145,9 @@ cell_get_contents(PyObject *self, void *closure) static int cell_set_contents(PyObject *self, PyObject *obj, void *Py_UNUSED(ignored)) { - PyCellObject *op = _PyCell_CAST(self); - Py_XSETREF(op->ob_ref, Py_XNewRef(obj)); + PyCellObject *cell = _PyCell_CAST(self); + Py_XINCREF(obj); + PyCell_SetTakeRef((PyCellObject *)cell, obj); return 0; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e96674c3502ef1..dd28aae6a3cb61 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1946,7 +1946,7 @@ dummy_func( }; specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -1955,7 +1955,7 @@ dummy_func( } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr, null if (oparg & 1))) { diff --git a/Python/ceval.c b/Python/ceval.c index f9514a6bf25c1b..6795a160506231 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -28,7 +28,6 @@ #include "pycore_setobject.h" // _PySet_Update() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_tuple.h" // _PyTuple_ITEMS() -#include "pycore_typeobject.h" // _PySuper_Lookup() #include "pycore_uop_ids.h" // Uops #include "pycore_pyerrors.h" diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e1f951558de7da..c31601f6d82b77 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6330,7 +6330,7 @@ global_super_st = stack_pointer[-3]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -6341,7 +6341,7 @@ } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _LOAD_SUPER_ATTR { diff --git a/Python/specialize.c b/Python/specialize.c index ba13b02a29b133..0fe4e7904de9f8 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -794,9 +794,8 @@ _Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _P PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *cls = PyStackRef_AsPyObjectBorrow(cls_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - _PySuperAttrCache *cache = (_PySuperAttrCache *)(instr + 1); if (global_super != (PyObject *)&PySuper_Type) { SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED); goto fail; @@ -805,19 +804,11 @@ _Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _P SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS); goto fail; } - instr->op.code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR; - goto success; - -fail: - STAT_INC(LOAD_SUPER_ATTR, failure); - assert(!PyErr_Occurred()); - instr->op.code = LOAD_SUPER_ATTR; - cache->counter = adaptive_counter_backoff(cache->counter); + uint8_t load_code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR; + specialize(instr, load_code); return; -success: - STAT_INC(LOAD_SUPER_ATTR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); +fail: + unspecialize(instr); } typedef enum { From 13b68e1a61e92a032d255aff5d5af435bbb63e8b Mon Sep 17 00:00:00 2001 From: mpage Date: Tue, 3 Dec 2024 09:50:58 -0800 Subject: [PATCH 089/427] gh-127421: Fix race in test_start_new_thread_failed (#127549) Fix race in test_start_new_thread_failed When we succeed in starting a new thread, for example if setrlimit was ineffective, we must wait for the newly spawned thread to exit. Otherwise, we run the risk that the newly spawned thread will race with runtime finalization and access memory that has already been clobbered/freed. `_thread.start_new_thread()` only spawns daemon threads, which the runtime does not wait for at shutdown, and does not return a handle. Use `_thread.start_joinable_thread()` and join the resulting handle when the thread is started successfully. --- Lib/test/test_threading.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index b666533466e578..fe225558fc4f0b 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1192,11 +1192,12 @@ def f(): resource.setrlimit(resource.RLIMIT_NPROC, (0, hard)) try: - _thread.start_new_thread(f, ()) + handle = _thread.start_joinable_thread(f) except RuntimeError: print('ok') else: print('!skip!') + handle.join() """ _, out, err = assert_python_ok("-u", "-c", code) out = out.strip() From 276cd66ccbbf85996a57bd1db3dd29b93a6eab64 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Tue, 3 Dec 2024 10:25:12 -0800 Subject: [PATCH 090/427] gh-115999: Add free-threaded specialization for `SEND` (gh-127426) No additional thread safety changes are required. Note that sending to a generator that is shared between threads is currently not safe in the free-threaded build. --- Lib/test/test_opcache.py | 42 ++++++++++++++++++++++++++++++++++++++ Python/bytecodes.c | 4 ++-- Python/generated_cases.c.h | 4 ++-- Python/specialize.c | 15 ++++---------- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 527114445ac13b..b7a18133ab8b8a 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1311,6 +1311,48 @@ def contains_op_set(): self.assert_specialized(contains_op_set, "CONTAINS_OP_SET") self.assert_no_opcode(contains_op_set, "CONTAINS_OP") + @cpython_only + @requires_specialization_ft + def test_send_with(self): + def run_async(coro): + while True: + try: + coro.send(None) + except StopIteration: + break + + class CM: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + pass + + async def send_with(): + for i in range(100): + async with CM(): + x = 1 + + run_async(send_with()) + # Note there are still unspecialized "SEND" opcodes in the + # cleanup paths of the 'with' statement. + self.assert_specialized(send_with, "SEND_GEN") + + @cpython_only + @requires_specialization_ft + def test_send_yield_from(self): + def g(): + yield None + + def send_yield_from(): + yield from g() + + for i in range(100): + list(send_yield_from()) + + self.assert_specialized(send_yield_from, "SEND_GEN") + self.assert_no_opcode(send_yield_from, "SEND") + @cpython_only @requires_specialization_ft def test_to_bool(self): diff --git a/Python/bytecodes.c b/Python/bytecodes.c index dd28aae6a3cb61..d6be3cebf80724 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1117,7 +1117,7 @@ dummy_func( }; specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); @@ -1125,7 +1125,7 @@ dummy_func( } OPCODE_DEFERRED_INC(SEND); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_SEND, (receiver, v -- receiver, retval)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c31601f6d82b77..ef191f6f697f24 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7065,7 +7065,7 @@ receiver = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7075,7 +7075,7 @@ } OPCODE_DEFERRED_INC(SEND); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _SEND { diff --git a/Python/specialize.c b/Python/specialize.c index 0fe4e7904de9f8..8b2d1a14c107e0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2627,28 +2627,21 @@ _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) { PyObject *receiver = PyStackRef_AsPyObjectBorrow(receiver_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND); - _PySendCache *cache = (_PySendCache *)(instr + 1); PyTypeObject *tp = Py_TYPE(receiver); if (tp == &PyGen_Type || tp == &PyCoro_Type) { if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(SEND, SPEC_FAIL_OTHER); goto failure; } - instr->op.code = SEND_GEN; - goto success; + specialize(instr, SEND_GEN); + return; } SPECIALIZATION_FAIL(SEND, _PySpecialization_ClassifyIterator(receiver)); failure: - STAT_INC(SEND, failure); - instr->op.code = SEND; - cache->counter = adaptive_counter_backoff(cache->counter); - return; -success: - STAT_INC(SEND, success); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr); } #ifdef Py_STATS From fc5a0dc22483a35068888e828c65796d7a792c14 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Tue, 3 Dec 2024 10:33:06 -0800 Subject: [PATCH 091/427] gh-127271: Replace use of PyCell_GET/SET (gh-127272) * Replace uses of `PyCell_GET` and `PyCell_SET`. These macros are not safe to use in the free-threaded build. Use `PyCell_GetRef()` and `PyCell_SetTakeRef()` instead. * Since `PyCell_GetRef()` returns a strong rather than borrowed ref, some code restructuring was required, e.g. `frame_get_var()` returns a strong ref now. * Add critical sections to `PyCell_GET` and `PyCell_SET`. * Move critical_section.h earlier in the Python.h file. * Add `PyCell_GET` to the free-threading howto table of APIs that return borrowed refs. * Add additional unit tests for free-threading. --- Doc/howto/free-threading-extensions.rst | 2 + Include/Python.h | 2 +- Include/cpython/cellobject.h | 8 +- Lib/test/test_free_threading/test_races.py | 134 +++++++++++++++++++++ Objects/cellobject.c | 41 ++++--- Objects/frameobject.c | 28 +++-- Objects/typeobject.c | 57 ++++++--- Python/bltinmodule.c | 7 +- 8 files changed, 231 insertions(+), 48 deletions(-) create mode 100644 Lib/test/test_free_threading/test_races.py diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 6abe93d71ad529..c1ad42e7e55ee5 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -167,6 +167,8 @@ that return :term:`strong references `. +-----------------------------------+-----------------------------------+ | :c:func:`PyImport_AddModule` | :c:func:`PyImport_AddModuleRef` | +-----------------------------------+-----------------------------------+ +| :c:func:`PyCell_GET` | :c:func:`PyCell_Get` | ++-----------------------------------+-----------------------------------+ Not all APIs that return borrowed references are problematic. For example, :c:func:`PyTuple_GetItem` is safe because tuples are immutable. diff --git a/Include/Python.h b/Include/Python.h index 717e27feab62db..64be80145890a3 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -69,6 +69,7 @@ #include "pystats.h" #include "pyatomic.h" #include "lock.h" +#include "critical_section.h" #include "object.h" #include "refcount.h" #include "objimpl.h" @@ -130,7 +131,6 @@ #include "import.h" #include "abstract.h" #include "bltinmodule.h" -#include "critical_section.h" #include "cpython/pyctype.h" #include "pystrtod.h" #include "pystrcmp.h" diff --git a/Include/cpython/cellobject.h b/Include/cpython/cellobject.h index 47a6a491497ea0..85a63a13747d87 100644 --- a/Include/cpython/cellobject.h +++ b/Include/cpython/cellobject.h @@ -22,10 +22,14 @@ PyAPI_FUNC(PyObject *) PyCell_Get(PyObject *); PyAPI_FUNC(int) PyCell_Set(PyObject *, PyObject *); static inline PyObject* PyCell_GET(PyObject *op) { + PyObject *res; PyCellObject *cell; assert(PyCell_Check(op)); cell = _Py_CAST(PyCellObject*, op); - return cell->ob_ref; + Py_BEGIN_CRITICAL_SECTION(cell); + res = cell->ob_ref; + Py_END_CRITICAL_SECTION(); + return res; } #define PyCell_GET(op) PyCell_GET(_PyObject_CAST(op)) @@ -33,7 +37,9 @@ static inline void PyCell_SET(PyObject *op, PyObject *value) { PyCellObject *cell; assert(PyCell_Check(op)); cell = _Py_CAST(PyCellObject*, op); + Py_BEGIN_CRITICAL_SECTION(cell); cell->ob_ref = value; + Py_END_CRITICAL_SECTION(); } #define PyCell_SET(op, value) PyCell_SET(_PyObject_CAST(op), (value)) diff --git a/Lib/test/test_free_threading/test_races.py b/Lib/test/test_free_threading/test_races.py new file mode 100644 index 00000000000000..09e1d52e3509f9 --- /dev/null +++ b/Lib/test/test_free_threading/test_races.py @@ -0,0 +1,134 @@ +# It's most useful to run these tests with ThreadSanitizer enabled. +import sys +import functools +import threading +import time +import unittest + +from test.support import threading_helper + + +class TestBase(unittest.TestCase): + pass + + +def do_race(func1, func2): + """Run func1() and func2() repeatedly in separate threads.""" + n = 1000 + + barrier = threading.Barrier(2) + + def repeat(func): + barrier.wait() + for _i in range(n): + func() + + threads = [ + threading.Thread(target=functools.partial(repeat, func1)), + threading.Thread(target=functools.partial(repeat, func2)), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +@threading_helper.requires_working_threading() +class TestRaces(TestBase): + def test_racing_cell_set(self): + """Test cell object gettr/settr properties.""" + + def nested_func(): + x = 0 + + def inner(): + nonlocal x + x += 1 + + # This doesn't race because LOAD_DEREF and STORE_DEREF on the + # cell object use critical sections. + do_race(nested_func, nested_func) + + def nested_func2(): + x = 0 + + def inner(): + y = x + frame = sys._getframe(1) + frame.f_locals["x"] = 2 + + return inner + + def mutate_func2(): + inner = nested_func2() + cell = inner.__closure__[0] + old_value = cell.cell_contents + cell.cell_contents = 1000 + time.sleep(0) + cell.cell_contents = old_value + time.sleep(0) + + # This revealed a race with cell_set_contents() since it was missing + # the critical section. + do_race(nested_func2, mutate_func2) + + def test_racing_cell_cmp_repr(self): + """Test cell object compare and repr methods.""" + + def nested_func(): + x = 0 + y = 0 + + def inner(): + return x + y + + return inner.__closure__ + + cell_a, cell_b = nested_func() + + def mutate(): + cell_a.cell_contents += 1 + + def access(): + cell_a == cell_b + s = repr(cell_a) + + # cell_richcompare() and cell_repr used to have data races + do_race(mutate, access) + + def test_racing_load_super_attr(self): + """Test (un)specialization of LOAD_SUPER_ATTR opcode.""" + + class C: + def __init__(self): + try: + super().__init__ + super().__init__() + except RuntimeError: + pass # happens if __class__ is replaced with non-type + + def access(): + C() + + def mutate(): + # Swap out the super() global with a different one + real_super = super + globals()["super"] = lambda s=1: s + time.sleep(0) + globals()["super"] = real_super + time.sleep(0) + # Swap out the __class__ closure value with a non-type + cell = C.__init__.__closure__[0] + real_class = cell.cell_contents + cell.cell_contents = 99 + time.sleep(0) + cell.cell_contents = real_class + + # The initial PR adding specialized opcodes for LOAD_SUPER_ATTR + # had some races (one with the super() global changing and one + # with the cell binding being changed). + do_race(access, mutate) + + +if __name__ == "__main__": + unittest.main() diff --git a/Objects/cellobject.c b/Objects/cellobject.c index 4ab9083af5e300..ec2eeb1a855b63 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -82,6 +82,17 @@ cell_dealloc(PyObject *self) PyObject_GC_Del(op); } +static PyObject * +cell_compare_impl(PyObject *a, PyObject *b, int op) +{ + if (a != NULL && b != NULL) { + return PyObject_RichCompare(a, b, op); + } + else { + Py_RETURN_RICHCOMPARE(b == NULL, a == NULL, op); + } +} + static PyObject * cell_richcompare(PyObject *a, PyObject *b, int op) { @@ -92,27 +103,28 @@ cell_richcompare(PyObject *a, PyObject *b, int op) if (!PyCell_Check(a) || !PyCell_Check(b)) { Py_RETURN_NOTIMPLEMENTED; } + PyObject *a_ref = PyCell_GetRef((PyCellObject *)a); + PyObject *b_ref = PyCell_GetRef((PyCellObject *)b); /* compare cells by contents; empty cells come before anything else */ - a = ((PyCellObject *)a)->ob_ref; - b = ((PyCellObject *)b)->ob_ref; - if (a != NULL && b != NULL) - return PyObject_RichCompare(a, b, op); + PyObject *res = cell_compare_impl(a_ref, b_ref, op); - Py_RETURN_RICHCOMPARE(b == NULL, a == NULL, op); + Py_XDECREF(a_ref); + Py_XDECREF(b_ref); + return res; } static PyObject * cell_repr(PyObject *self) { - PyCellObject *op = _PyCell_CAST(self); - if (op->ob_ref == NULL) { - return PyUnicode_FromFormat("", op); + PyObject *ref = PyCell_GetRef((PyCellObject *)self); + if (ref == NULL) { + return PyUnicode_FromFormat("", self); } - - return PyUnicode_FromFormat("", - op, Py_TYPE(op->ob_ref)->tp_name, - op->ob_ref); + PyObject *res = PyUnicode_FromFormat("", + self, Py_TYPE(ref)->tp_name, ref); + Py_DECREF(ref); + return res; } static int @@ -135,11 +147,12 @@ static PyObject * cell_get_contents(PyObject *self, void *closure) { PyCellObject *op = _PyCell_CAST(self); - if (op->ob_ref == NULL) { + PyObject *res = PyCell_GetRef(op); + if (res == NULL) { PyErr_SetString(PyExc_ValueError, "Cell is empty"); return NULL; } - return Py_NewRef(op->ob_ref); + return res; } static int diff --git a/Objects/frameobject.c b/Objects/frameobject.c index c743c254848d3a..03ed2b9480f8c9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -8,6 +8,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetDict() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_cell.h" // PyCell_GetRef() PyCell_SetTakeRef() #include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches @@ -187,11 +188,8 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) } } if (cell != NULL) { - PyObject *oldvalue_o = PyCell_GET(cell); - if (value != oldvalue_o) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue_o); - } + Py_XINCREF(value); + PyCell_SetTakeRef((PyCellObject *)cell, value); } else if (value != PyStackRef_AsPyObjectBorrow(oldvalue)) { PyStackRef_XCLOSE(fast[i]); fast[i] = PyStackRef_FromPyObjectNew(value); @@ -1987,19 +1985,25 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, if (kind & CO_FAST_FREE) { // The cell was set by COPY_FREE_VARS. assert(value != NULL && PyCell_Check(value)); - value = PyCell_GET(value); + value = PyCell_GetRef((PyCellObject *)value); } else if (kind & CO_FAST_CELL) { if (value != NULL) { if (PyCell_Check(value)) { assert(!_PyFrame_IsIncomplete(frame)); - value = PyCell_GET(value); + value = PyCell_GetRef((PyCellObject *)value); + } + else { + // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), + // with the initial value set when the frame was created... + // (unlikely) ...or it was set via the f_locals proxy. + Py_INCREF(value); } - // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), - // with the initial value set when the frame was created... - // (unlikely) ...or it was set via the f_locals proxy. } } + else { + Py_XINCREF(value); + } } *pvalue = value; return 1; @@ -2076,14 +2080,14 @@ PyFrame_GetVar(PyFrameObject *frame_obj, PyObject *name) continue; } - PyObject *value; // borrowed reference + PyObject *value; if (!frame_get_var(frame, co, i, &value)) { break; } if (value == NULL) { break; } - return Py_NewRef(value); + return value; } PyErr_Format(PyExc_NameError, "variable %R does not exist", name); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2611404a3d0d61..bf9049bce3adeb 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -19,6 +19,7 @@ #include "pycore_typeobject.h" // struct type_cache #include "pycore_unionobject.h" // _Py_union_type_or #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "pycore_cell.h" // PyCell_GetRef() #include "opcode.h" // MAKE_CELL #include // ptrdiff_t @@ -11676,23 +11677,28 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, assert(_PyFrame_GetCode(cframe)->co_nlocalsplus > 0); PyObject *firstarg = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[0]); + if (firstarg == NULL) { + PyErr_SetString(PyExc_RuntimeError, "super(): arg[0] deleted"); + return -1; + } // The first argument might be a cell. - if (firstarg != NULL && (_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) { - // "firstarg" is a cell here unless (very unlikely) super() - // was called from the C-API before the first MAKE_CELL op. - if (_PyInterpreterFrame_LASTI(cframe) >= 0) { - // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need - // to use _PyOpcode_Deopt here: - assert(_PyCode_CODE(co)[0].op.code == MAKE_CELL || - _PyCode_CODE(co)[0].op.code == COPY_FREE_VARS); - assert(PyCell_Check(firstarg)); - firstarg = PyCell_GET(firstarg); + // "firstarg" is a cell here unless (very unlikely) super() + // was called from the C-API before the first MAKE_CELL op. + if ((_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL) && + (_PyInterpreterFrame_LASTI(cframe) >= 0)) { + // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need + // to use _PyOpcode_Deopt here: + assert(_PyCode_CODE(co)[0].op.code == MAKE_CELL || + _PyCode_CODE(co)[0].op.code == COPY_FREE_VARS); + assert(PyCell_Check(firstarg)); + firstarg = PyCell_GetRef((PyCellObject *)firstarg); + if (firstarg == NULL) { + PyErr_SetString(PyExc_RuntimeError, "super(): arg[0] deleted"); + return -1; } } - if (firstarg == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "super(): arg[0] deleted"); - return -1; + else { + Py_INCREF(firstarg); } // Look for __class__ in the free vars. @@ -11707,18 +11713,22 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, if (cell == NULL || !PyCell_Check(cell)) { PyErr_SetString(PyExc_RuntimeError, "super(): bad __class__ cell"); + Py_DECREF(firstarg); return -1; } - type = (PyTypeObject *) PyCell_GET(cell); + type = (PyTypeObject *) PyCell_GetRef((PyCellObject *)cell); if (type == NULL) { PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); + Py_DECREF(firstarg); return -1; } if (!PyType_Check(type)) { PyErr_Format(PyExc_RuntimeError, "super(): __class__ is not a type (%s)", Py_TYPE(type)->tp_name); + Py_DECREF(type); + Py_DECREF(firstarg); return -1; } break; @@ -11727,6 +11737,7 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, if (type == NULL) { PyErr_SetString(PyExc_RuntimeError, "super(): __class__ cell not found"); + Py_DECREF(firstarg); return -1; } @@ -11773,16 +11784,24 @@ super_init_impl(PyObject *self, PyTypeObject *type, PyObject *obj) { return -1; } } + else { + Py_INCREF(type); + Py_XINCREF(obj); + } - if (obj == Py_None) + if (obj == Py_None) { + Py_DECREF(obj); obj = NULL; + } if (obj != NULL) { obj_type = supercheck(type, obj); - if (obj_type == NULL) + if (obj_type == NULL) { + Py_DECREF(type); + Py_DECREF(obj); return -1; - Py_INCREF(obj); + } } - Py_XSETREF(su->type, (PyTypeObject*)Py_NewRef(type)); + Py_XSETREF(su->type, (PyTypeObject*)type); Py_XSETREF(su->obj, obj); Py_XSETREF(su->obj_type, obj_type); return 0; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 17df9208f224f4..fb9868b3740b8c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -13,6 +13,7 @@ #include "pycore_pythonrun.h" // _Py_SourceAsString() #include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_cell.h" // PyCell_GetRef() #include "clinic/bltinmodule.c.h" @@ -209,7 +210,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *margs[3] = {name, bases, ns}; cls = PyObject_VectorcallDict(meta, margs, 3, mkw); if (cls != NULL && PyType_Check(cls) && PyCell_Check(cell)) { - PyObject *cell_cls = PyCell_GET(cell); + PyObject *cell_cls = PyCell_GetRef((PyCellObject *)cell); if (cell_cls != cls) { if (cell_cls == NULL) { const char *msg = @@ -221,9 +222,13 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, "__class__ set to %.200R defining %.200R as %.200R"; PyErr_Format(PyExc_TypeError, msg, cell_cls, name, cls); } + Py_XDECREF(cell_cls); Py_SETREF(cls, NULL); goto error; } + else { + Py_DECREF(cell_cls); + } } } error: From dabcecfd6dadb9430733105ba36925b290343d31 Mon Sep 17 00:00:00 2001 From: mpage Date: Tue, 3 Dec 2024 11:20:20 -0800 Subject: [PATCH 092/427] gh-115999: Enable specialization of `CALL` instructions in free-threaded builds (#127123) The CALL family of instructions were mostly thread-safe already and only required a small number of changes, which are documented below. A few changes were needed to make CALL_ALLOC_AND_ENTER_INIT thread-safe: Added _PyType_LookupRefAndVersion, which returns the type version corresponding to the returned ref. Added _PyType_CacheInitForSpecialization, which takes an init method and the corresponding type version and only populates the specialization cache if the current type version matches the supplied version. This prevents potentially caching a stale value in free-threaded builds if we race with an update to __init__. Only cache __init__ functions that are deferred in free-threaded builds. This ensures that the reference to __init__ that is stored in the specialization cache is valid if the type version guard in _CHECK_AND_ALLOCATE_OBJECT passes. Fix a bug in _CREATE_INIT_FRAME where the frame is pushed to the stack on failure. A few other miscellaneous changes were also needed: Use {LOCK,UNLOCK}_OBJECT in LIST_APPEND. This ensures that the list's per-object lock is held while we are appending to it. Add missing co_tlbc for _Py_InitCleanup. Stop/start the world around setting the eval frame hook. This allows us to read interp->eval_frame non-atomically and preserves the behavior of _CHECK_PEP_523 documented below. --- Include/internal/pycore_object.h | 14 ++++ Lib/test/test_monitoring.py | 17 +++-- Lib/test/test_opcache.py | 32 +++++++-- Lib/test/test_type_cache.py | 9 ++- Objects/typeobject.c | 62 +++++++++++++++-- Python/bytecodes.c | 16 +++-- Python/executor_cases.c.h | 21 ++++-- Python/generated_cases.c.h | 21 +++--- Python/perf_trampoline.c | 6 +- Python/pystate.c | 2 + Python/specialize.c | 112 ++++++++++++++++++------------- 11 files changed, 220 insertions(+), 92 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index c52ed8f14707ba..ce876b093b2522 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -835,6 +835,20 @@ extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value); extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr); +extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, + unsigned int *); + +// Cache the provided init method in the specialization cache of type if the +// provided type version matches the current version of the type. +// +// The cached value is borrowed and is only valid if guarded by a type +// version check. In free-threaded builds the init method must also use +// deferred reference counting. +// +// Returns 1 if the value was cached or 0 otherwise. +extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, + PyObject *init, + unsigned int tp_version); #ifdef Py_GIL_DISABLED # define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index b640aa08e4a812..5a4bcebedf19de 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -11,7 +11,7 @@ import unittest import test.support -from test.support import requires_specialization, script_helper +from test.support import requires_specialization_ft, script_helper from test.support.import_helper import import_module _testcapi = test.support.import_helper.import_module("_testcapi") @@ -850,6 +850,13 @@ def __init__(self, events): def __call__(self, code, offset, val): self.events.append(("return", code.co_name, val)) +# gh-127274: CALL_ALLOC_AND_ENTER_INIT will only cache __init__ methods that +# are deferred. We only defer functions defined at the top-level. +class ValueErrorRaiser: + def __init__(self): + raise ValueError() + + class ExceptionMonitoringTest(CheckEvents): exception_recorders = ( @@ -1045,16 +1052,12 @@ def func(): ) self.assertEqual(events[0], ("throw", IndexError)) - @requires_specialization + @requires_specialization_ft def test_no_unwind_for_shim_frame(self): - class B: - def __init__(self): - raise ValueError() - def f(): try: - return B() + return ValueErrorRaiser() except ValueError: pass diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index b7a18133ab8b8a..50b5f365165921 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -493,6 +493,18 @@ def f(): self.assertFalse(f()) +# gh-127274: CALL_ALLOC_AND_ENTER_INIT will only cache __init__ methods that +# are deferred. We only defer functions defined at the top-level. +class MyClass: + def __init__(self): + pass + + +class InitTakesArg: + def __init__(self, arg): + self.arg = arg + + class TestCallCache(TestBase): def test_too_many_defaults_0(self): def f(): @@ -522,12 +534,8 @@ def f(x, y): f() @disabling_optimizer - @requires_specialization + @requires_specialization_ft def test_assign_init_code(self): - class MyClass: - def __init__(self): - pass - def instantiate(): return MyClass() @@ -544,6 +552,20 @@ def count_args(self, *args): MyClass.__init__.__code__ = count_args.__code__ instantiate() + @disabling_optimizer + @requires_specialization_ft + def test_push_init_frame_fails(self): + def instantiate(): + return InitTakesArg() + + for _ in range(2): + with self.assertRaises(TypeError): + instantiate() + self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") + + with self.assertRaises(TypeError): + instantiate() + @threading_helper.requires_working_threading() class TestRacesDoNotCrash(TestBase): diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 66abe73f8d766d..e109a65741309a 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -2,7 +2,7 @@ import unittest import dis from test import support -from test.support import import_helper, requires_specialization +from test.support import import_helper, requires_specialization, requires_specialization_ft try: from sys import _clear_type_cache except ImportError: @@ -110,7 +110,6 @@ class HolderSub(Holder): HolderSub.value @support.cpython_only -@requires_specialization class TypeCacheWithSpecializationTests(unittest.TestCase): def tearDown(self): _clear_type_cache() @@ -140,6 +139,7 @@ def _check_specialization(self, func, arg, opname, *, should_specialize): else: self.assertIn(opname, self._all_opnames(func)) + @requires_specialization def test_class_load_attr_specialization_user_type(self): class A: def foo(self): @@ -160,6 +160,7 @@ def load_foo_2(type_): self._check_specialization(load_foo_2, A, "LOAD_ATTR", should_specialize=False) + @requires_specialization def test_class_load_attr_specialization_static_type(self): self.assertNotEqual(type_get_version(str), 0) self.assertNotEqual(type_get_version(bytes), 0) @@ -171,6 +172,7 @@ def get_capitalize_1(type_): self.assertEqual(get_capitalize_1(str)('hello'), 'Hello') self.assertEqual(get_capitalize_1(bytes)(b'hello'), b'Hello') + @requires_specialization def test_property_load_attr_specialization_user_type(self): class G: @property @@ -192,6 +194,7 @@ def load_x_2(instance): self._check_specialization(load_x_2, G(), "LOAD_ATTR", should_specialize=False) + @requires_specialization def test_store_attr_specialization_user_type(self): class B: __slots__ = ("bar",) @@ -211,6 +214,7 @@ def store_bar_2(type_): self._check_specialization(store_bar_2, B(), "STORE_ATTR", should_specialize=False) + @requires_specialization_ft def test_class_call_specialization_user_type(self): class F: def __init__(self): @@ -231,6 +235,7 @@ def call_class_2(type_): self._check_specialization(call_class_2, F, "CALL", should_specialize=False) + @requires_specialization def test_to_bool_specialization_user_type(self): class H: pass diff --git a/Objects/typeobject.c b/Objects/typeobject.c index bf9049bce3adeb..2068d6aa9be52b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5528,9 +5528,12 @@ _PyTypes_AfterFork(void) } /* Internal API to look for a name through the MRO. - This returns a borrowed reference, and doesn't set an exception! */ + This returns a strong reference, and doesn't set an exception! + If nonzero, version is set to the value of type->tp_version at the time of + the lookup. +*/ PyObject * -_PyType_LookupRef(PyTypeObject *type, PyObject *name) +_PyType_LookupRefAndVersion(PyTypeObject *type, PyObject *name, unsigned int *version) { PyObject *res; int error; @@ -5553,6 +5556,9 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name) // If the sequence is still valid then we're done if (value == NULL || _Py_TryIncref(value)) { if (_PySeqLock_EndRead(&entry->sequence, sequence)) { + if (version != NULL) { + *version = entry_version; + } return value; } Py_XDECREF(value); @@ -5574,6 +5580,9 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name) OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); Py_XINCREF(entry->value); + if (version != NULL) { + *version = entry->version; + } return entry->value; } #endif @@ -5587,12 +5596,12 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name) // anyone else can modify our mro or mutate the type. int has_version = 0; - int version = 0; + unsigned int assigned_version = 0; BEGIN_TYPE_LOCK(); res = find_name_in_mro(type, name, &error); if (MCACHE_CACHEABLE_NAME(name)) { has_version = assign_version_tag(interp, type); - version = type->tp_version_tag; + assigned_version = type->tp_version_tag; } END_TYPE_LOCK(); @@ -5609,28 +5618,67 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name) if (error == -1) { PyErr_Clear(); } + if (version != NULL) { + // 0 is not a valid version + *version = 0; + } return NULL; } if (has_version) { #if Py_GIL_DISABLED - update_cache_gil_disabled(entry, name, version, res); + update_cache_gil_disabled(entry, name, assigned_version, res); #else - PyObject *old_value = update_cache(entry, name, version, res); + PyObject *old_value = update_cache(entry, name, assigned_version, res); Py_DECREF(old_value); #endif } + if (version != NULL) { + // 0 is not a valid version + *version = has_version ? assigned_version : 0; + } return res; } +/* Internal API to look for a name through the MRO. + This returns a strong reference, and doesn't set an exception! +*/ +PyObject * +_PyType_LookupRef(PyTypeObject *type, PyObject *name) +{ + return _PyType_LookupRefAndVersion(type, name, NULL); +} + +/* Internal API to look for a name through the MRO. + This returns a borrowed reference, and doesn't set an exception! */ PyObject * _PyType_Lookup(PyTypeObject *type, PyObject *name) { - PyObject *res = _PyType_LookupRef(type, name); + PyObject *res = _PyType_LookupRefAndVersion(type, name, NULL); Py_XDECREF(res); return res; } +int +_PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init, + unsigned int tp_version) +{ + if (!init || !tp_version) { + return 0; + } + int can_cache; + BEGIN_TYPE_LOCK(); + can_cache = ((PyTypeObject*)type)->tp_version_tag == tp_version; + #ifdef Py_GIL_DISABLED + can_cache = can_cache && _PyObject_HasDeferredRefcount(init); + #endif + if (can_cache) { + FT_ATOMIC_STORE_PTR_RELEASE(type->_spec_cache.init, init); + } + END_TYPE_LOCK(); + return can_cache; +} + static void set_flags(PyTypeObject *self, unsigned long mask, unsigned long flags) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d6be3cebf80724..3d280941b35244 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3329,7 +3329,7 @@ dummy_func( }; specializing op(_SPECIALIZE_CALL, (counter/1, callable[1], self_or_null[1], args[oparg] -- callable[1], self_or_null[1], args[oparg])) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); @@ -3337,7 +3337,7 @@ dummy_func( } OPCODE_DEFERRED_INC(CALL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_MAYBE_EXPAND_METHOD, (callable[1], self_or_null[1], args[oparg] -- func[1], maybe_self[1], args[oparg])) { @@ -3722,10 +3722,10 @@ dummy_func( DEOPT_IF(!PyStackRef_IsNull(null[0])); DEOPT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_version_tag != type_version); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version); assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)); STAT_INC(CALL, hit); @@ -3743,17 +3743,19 @@ dummy_func( _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); DEAD(init); DEAD(self); - init_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init[0], NULL, args-1, oparg+1, NULL, shim); SYNC_SP(); - if (init_frame == NULL) { + if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); ERROR_NO_POP(); } + init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; /* Account for pushing the extra frame. * We don't check recursion depth here, @@ -4000,8 +4002,10 @@ dummy_func( DEOPT_IF(callable_o != interp->callable_cache.list_append); assert(self_o != NULL); DEOPT_IF(!PyList_Check(self_o)); + DEOPT_IF(!LOCK_OBJECT(self_o)); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); ERROR_IF(err, error); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 580814657608db..987ff2e6419669 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4500,13 +4500,13 @@ JUMP_TO_JUMP_TARGET(); } PyTypeObject *tp = (PyTypeObject *)callable_o; - if (tp->tp_version_tag != type_version) { + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UOP_STAT_INC(uopcode, miss); @@ -4537,25 +4537,29 @@ _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); stack_pointer = _PyFrame_GetStackPointer(frame); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); _PyFrame_SetStackPointer(frame, stack_pointer); - init_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init[0], NULL, args-1, oparg+1, NULL, shim); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-2 - oparg].bits = (uintptr_t)init_frame; - stack_pointer += -1 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (init_frame == NULL) { + if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); JUMP_TO_ERROR(); } + init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; /* Account for pushing the extra frame. * We don't check recursion depth here, * as it will be checked after start_frame */ tstate->py_recursion_remaining--; + stack_pointer[0].bits = (uintptr_t)init_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -4908,8 +4912,13 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + if (!LOCK_OBJECT(self_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); if (err) JUMP_TO_ERROR(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ef191f6f697f24..33f32aba1e5145 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -880,7 +880,7 @@ callable = &stack_pointer[-2 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -890,7 +890,7 @@ } OPCODE_DEFERRED_INC(CALL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD @@ -1048,10 +1048,10 @@ DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); DEOPT_IF(!PyType_Check(callable_o), CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_version_tag != type_version, CALL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); STAT_INC(CALL, hit); @@ -1073,20 +1073,21 @@ _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); stack_pointer = _PyFrame_GetStackPointer(frame); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); _PyFrame_SetStackPointer(frame, stack_pointer); - init_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init[0], NULL, args-1, oparg+1, NULL, shim); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-2 - oparg].bits = (uintptr_t)init_frame; - stack_pointer += -1 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (init_frame == NULL) { + if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); goto error; } + init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; /* Account for pushing the extra frame. * We don't check recursion depth here, @@ -1100,8 +1101,6 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -2383,8 +2382,10 @@ DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); assert(self_o != NULL); DEOPT_IF(!PyList_Check(self_o), CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); if (err) goto pop_3_error; diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index f144f7d436fe68..ad077dc861b0a7 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -484,11 +484,11 @@ _PyPerfTrampoline_Init(int activate) return -1; } if (!activate) { - tstate->interp->eval_frame = NULL; + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, NULL); perf_status = PERF_STATUS_NO_INIT; } else { - tstate->interp->eval_frame = py_trampoline_evaluator; + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, py_trampoline_evaluator); if (new_code_arena() < 0) { return -1; } @@ -514,7 +514,7 @@ _PyPerfTrampoline_Fini(void) } PyThreadState *tstate = _PyThreadState_GET(); if (tstate->interp->eval_frame == py_trampoline_evaluator) { - tstate->interp->eval_frame = NULL; + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, NULL); } if (perf_status == PERF_STATUS_OK) { trampoline_api.free_state(trampoline_api.state); diff --git a/Python/pystate.c b/Python/pystate.c index 3ceae229f75cd0..839413a65a42fb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2838,7 +2838,9 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, } #endif RARE_EVENT_INC(set_eval_frame_func); + _PyEval_StopTheWorld(interp); interp->eval_frame = eval_frame; + _PyEval_StartTheWorld(interp); } diff --git a/Python/specialize.c b/Python/specialize.c index 8b2d1a14c107e0..ec2cd7025e5054 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1911,38 +1911,38 @@ _Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_COD unspecialize(instr); } -/* Returns a borrowed reference. - * The reference is only valid if guarded by a type version check. - */ -static PyFunctionObject * -get_init_for_simple_managed_python_class(PyTypeObject *tp) +/* Returns a strong reference. */ +static PyObject * +get_init_for_simple_managed_python_class(PyTypeObject *tp, unsigned int *tp_version) { assert(tp->tp_new == PyBaseObject_Type.tp_new); if (tp->tp_alloc != PyType_GenericAlloc) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN); return NULL; } - if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + unsigned long tp_flags = PyType_GetFlags(tp); + if ((tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES); return NULL; } - if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + if (!(tp_flags & Py_TPFLAGS_HEAPTYPE)) { /* Is this possible? */ SPECIALIZATION_FAIL(CALL, SPEC_FAIL_EXPECTED_ERROR); return NULL; } - PyObject *init = _PyType_Lookup(tp, &_Py_ID(__init__)); + PyObject *init = _PyType_LookupRefAndVersion(tp, &_Py_ID(__init__), tp_version); if (init == NULL || !PyFunction_Check(init)) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_PYTHON); + Py_XDECREF(init); return NULL; } int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); if (kind != SIMPLE_FUNCTION) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_SIMPLE); + Py_DECREF(init); return NULL; } - ((PyHeapTypeObject *)tp)->_spec_cache.init = init; - return (PyFunctionObject *)init; + return init; } static int @@ -1954,20 +1954,20 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) int oparg = instr->op.arg; if (nargs == 1 && oparg == 1) { if (tp == &PyUnicode_Type) { - instr->op.code = CALL_STR_1; + specialize(instr, CALL_STR_1); return 0; } else if (tp == &PyType_Type) { - instr->op.code = CALL_TYPE_1; + specialize(instr, CALL_TYPE_1); return 0; } else if (tp == &PyTuple_Type) { - instr->op.code = CALL_TUPLE_1; + specialize(instr, CALL_TUPLE_1); return 0; } } if (tp->tp_vectorcall != NULL) { - instr->op.code = CALL_BUILTIN_CLASS; + specialize(instr, CALL_BUILTIN_CLASS); return 0; } goto generic; @@ -1976,19 +1976,25 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) goto generic; } if (tp->tp_new == PyBaseObject_Type.tp_new) { - PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); - if (type_get_version(tp, CALL) == 0) { + unsigned int tp_version = 0; + PyObject *init = get_init_for_simple_managed_python_class(tp, &tp_version); + if (!tp_version) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(init); return -1; } - if (init != NULL) { + if (init != NULL && _PyType_CacheInitForSpecialization( + (PyHeapTypeObject *)tp, init, tp_version)) { _PyCallCache *cache = (_PyCallCache *)(instr + 1); - write_u32(cache->func_version, tp->tp_version_tag); - _Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT); + write_u32(cache->func_version, tp_version); + specialize(instr, CALL_ALLOC_AND_ENTER_INIT); + Py_DECREF(init); return 0; } + Py_XDECREF(init); } generic: - instr->op.code = CALL_NON_PY_GENERAL; + specialize(instr, CALL_NON_PY_GENERAL); return 0; } @@ -2004,7 +2010,7 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } - instr->op.code = CALL_METHOD_DESCRIPTOR_NOARGS; + specialize(instr, CALL_METHOD_DESCRIPTOR_NOARGS); return 0; } case METH_O: { @@ -2018,22 +2024,22 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (next.op.code == POP_TOP); int oparg = instr->op.arg; if ((PyObject *)descr == list_append && oparg == 1 && pop) { - instr->op.code = CALL_LIST_APPEND; + specialize(instr, CALL_LIST_APPEND); return 0; } - instr->op.code = CALL_METHOD_DESCRIPTOR_O; + specialize(instr, CALL_METHOD_DESCRIPTOR_O); return 0; } case METH_FASTCALL: { - instr->op.code = CALL_METHOD_DESCRIPTOR_FAST; + specialize(instr, CALL_METHOD_DESCRIPTOR_FAST); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - instr->op.code = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + specialize(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); return 0; } } - instr->op.code = CALL_NON_PY_GENERAL; + specialize(instr, CALL_NON_PY_GENERAL); return 0; } @@ -2063,12 +2069,15 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } write_u32(cache->func_version, version); + uint8_t opcode; if (argcount == nargs + bound_method) { - instr->op.code = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; + opcode = + bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; } else { - instr->op.code = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; + opcode = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; } + specialize(instr, opcode); return 0; } @@ -2117,10 +2126,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) /* len(o) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.len) { - instr->op.code = CALL_LEN; + specialize(instr, CALL_LEN); return 0; } - instr->op.code = CALL_BUILTIN_O; + specialize(instr, CALL_BUILTIN_O); return 0; } case METH_FASTCALL: { @@ -2128,19 +2137,19 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.isinstance) { - instr->op.code = CALL_ISINSTANCE; + specialize(instr, CALL_ISINSTANCE); return 0; } } - instr->op.code = CALL_BUILTIN_FAST; + specialize(instr, CALL_BUILTIN_FAST); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - instr->op.code = CALL_BUILTIN_FAST_WITH_KEYWORDS; + specialize(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); return 0; } default: - instr->op.code = CALL_NON_PY_GENERAL; + specialize(instr, CALL_NON_PY_GENERAL); return 0; } } @@ -2150,10 +2159,9 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); - _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; if (PyCFunction_CheckExact(callable)) { fail = specialize_c_call(callable, instr, nargs); @@ -2178,19 +2186,11 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) } } else { - instr->op.code = CALL_NON_PY_GENERAL; + specialize(instr, CALL_NON_PY_GENERAL); fail = 0; } if (fail) { - STAT_INC(CALL, failure); - assert(!PyErr_Occurred()); - instr->op.code = CALL; - cache->counter = adaptive_counter_backoff(cache->counter); - } - else { - STAT_INC(CALL, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr); } } @@ -2793,6 +2793,16 @@ _Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr) * Ends with a RESUME so that it is not traced. * This is used as a plain code object, not a function, * so must not access globals or builtins. + * There are a few other constraints imposed on the code + * by the free-threaded build: + * + * 1. The RESUME instruction must not be executed. Otherwise we may attempt to + * free the statically allocated TLBC array. + * 2. It must contain no specializable instructions. Specializing multiple + * copies of the same bytecode is not thread-safe in free-threaded builds. + * + * This should be dynamically allocated if either of those restrictions need to + * be lifted. */ #define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) @@ -2802,6 +2812,13 @@ static const PyBytesObject no_location = { .ob_sval = { NO_LOC_4 } }; +#ifdef Py_GIL_DISABLED +static _PyCodeArray init_cleanup_tlbc = { + .size = 1, + .entries = {(char*) &_Py_InitCleanup.co_code_adaptive}, +}; +#endif + const struct _PyCode8 _Py_InitCleanup = { _PyVarObject_HEAD_INIT(&PyCode_Type, 3), .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), @@ -2817,6 +2834,9 @@ const struct _PyCode8 _Py_InitCleanup = { ._co_firsttraceable = 4, .co_stacksize = 2, .co_framesize = 2 + FRAME_SPECIALS_SIZE, +#ifdef Py_GIL_DISABLED + .co_tlbc = &init_cleanup_tlbc, +#endif .co_code_adaptive = { EXIT_INIT_CHECK, 0, RETURN_VALUE, 0, From 12397a5781664bf43da98454db07cdfdec3ab815 Mon Sep 17 00:00:00 2001 From: "RUANG (James Roy)" Date: Wed, 4 Dec 2024 06:33:13 +0800 Subject: [PATCH 093/427] gh-112192: Increase the trace module coverage precision to one decimal (#126972) --- Lib/test/test_regrtest.py | 2 +- Lib/test/test_trace.py | 4 ++-- Lib/trace.py | 7 +++---- .../Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst | 1 + 4 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index d4f4a69a7a38c1..0ab7a23aca1df8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1138,7 +1138,7 @@ def test_coverage(self): output = self.run_tests("--coverage", test) self.check_executed_tests(output, [test], stats=1) regex = (r'lines +cov% +module +\(path\)\n' - r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') + r'(?: *[0-9]+ *[0-9]{1,2}\.[0-9]% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) def test_wait(self): diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 93966ee31d0a01..e7e42531916d0d 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -412,7 +412,7 @@ def test_issue9936(self): coverage = {} for line in stdout: lines, cov, module = line.split()[:3] - coverage[module] = (int(lines), int(cov[:-1])) + coverage[module] = (float(lines), float(cov[:-1])) # XXX This is needed to run regrtest.py as a script modname = trace._fullmodname(sys.modules[modname].__file__) self.assertIn(modname, coverage) @@ -553,7 +553,7 @@ def f(): stdout = stdout.decode() self.assertEqual(status, 0) self.assertIn('lines cov% module (path)', stdout) - self.assertIn(f'6 100% {modulename} ({filename})', stdout) + self.assertIn(f'6 100.0% {modulename} ({filename})', stdout) def test_run_as_module(self): assert_python_ok('-m', 'trace', '-l', '--module', 'timeit', '-n', '1') diff --git a/Lib/trace.py b/Lib/trace.py index bb3d34fd8d6550..a87bc6d61a884f 100644 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -279,14 +279,13 @@ def write_results(self, show_missing=True, summary=False, coverdir=None, *, n_hits, n_lines = self.write_results_file(coverpath, source, lnotab, count, encoding) if summary and n_lines: - percent = int(100 * n_hits / n_lines) - sums[modulename] = n_lines, percent, modulename, filename + sums[modulename] = n_lines, n_hits, modulename, filename if summary and sums: print("lines cov% module (path)") for m in sorted(sums): - n_lines, percent, modulename, filename = sums[m] - print("%5d %3d%% %s (%s)" % sums[m]) + n_lines, n_hits, modulename, filename = sums[m] + print(f"{n_lines:5d} {n_hits/n_lines:.1%} {modulename} ({filename})") if self.outfile: # try and store counts and module info into self.outfile diff --git a/Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst b/Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst new file mode 100644 index 00000000000000..b169c1508d0d30 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst @@ -0,0 +1 @@ +In the :mod:`trace` module, increase the coverage precision (``cov%``) to one decimal. From 0f9107817022f0defac157e3795a4093a32ea320 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 4 Dec 2024 01:06:20 +0100 Subject: [PATCH 094/427] gh-127146: Resolve some minor problems in Emscripten tests (#127565) Adjusts some Emscripten test exclusions regarding strftime, fma, and stack depth. --- Lib/test/test_marshal.py | 4 ++-- Lib/test/test_math.py | 2 +- Lib/test/test_support.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 93b8684c725d24..4ed9f1fc1b8020 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -1,5 +1,5 @@ from test import support -from test.support import is_apple_mobile, os_helper, requires_debug_ranges +from test.support import is_apple_mobile, os_helper, requires_debug_ranges, is_emscripten from test.support.script_helper import assert_python_ok import array import io @@ -294,7 +294,7 @@ def test_recursion_limit(self): #if os.name == 'nt' and support.Py_DEBUG: if os.name == 'nt': MAX_MARSHAL_STACK_DEPTH = 1000 - elif sys.platform == 'wasi' or is_apple_mobile: + elif sys.platform == 'wasi' or is_emscripten or is_apple_mobile: MAX_MARSHAL_STACK_DEPTH = 1500 else: MAX_MARSHAL_STACK_DEPTH = 2000 diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index fecafd53aa6e6f..6976a5d85da019 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -2722,7 +2722,7 @@ def test_fma_infinities(self): # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 # properly: it doesn't use the right sign when the result is zero. @unittest.skipIf( - sys.platform.startswith(("freebsd", "wasi", "netbsd")) + sys.platform.startswith(("freebsd", "wasi", "netbsd", "emscripten")) or (sys.platform == "android" and platform.machine() == "x86_64"), f"this platform doesn't implement IEE 754-2008 properly") def test_fma_zero_result(self): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 9a3cf140d81241..635ae03a404988 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -578,7 +578,7 @@ def test_print_warning(self): 'Warning -- a\nWarning -- b\n') def test_has_strftime_extensions(self): - if support.is_emscripten or sys.platform == "win32": + if sys.platform == "win32": self.assertFalse(support.has_strftime_extensions) else: self.assertTrue(support.has_strftime_extensions) From 7f882c88cfda486947974cb82c20a1ae7047edfc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 3 Dec 2024 18:20:01 -0600 Subject: [PATCH 095/427] Itertool recipe additions (gh-127483) --- Doc/library/itertools.rst | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index c138e903fa5a0f..03966f3d3d694b 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -877,6 +877,11 @@ and :term:`generators ` which incur interpreter overhead. "Returns the sequence elements n times." return chain.from_iterable(repeat(tuple(iterable), n)) + def loops(n): + "Loop n times. Like range(n) but without creating integers." + # for _ in loops(100): ... + return repeat(None, n) + def tail(n, iterable): "Return an iterator over the last n items." # tail(3, 'ABCDEFG') → E F G @@ -1099,6 +1104,11 @@ The following recipes have a more mathematical flavor: data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) yield from iter_index(data, 1, start=3) + def is_prime(n): + "Return True if n is prime." + # is_prime(1_000_000_000_000_403) → True + return n > 1 and all(n % p for p in sieve(math.isqrt(n) + 1)) + def factor(n): "Prime factors of n." # factor(99) → 3 3 11 @@ -1202,6 +1212,16 @@ The following recipes have a more mathematical flavor: [0, 2, 4, 6] + >>> for _ in loops(5): + ... print('hi') + ... + hi + hi + hi + hi + hi + + >>> list(tail(3, 'ABCDEFG')) ['E', 'F', 'G'] >>> # Verify the input is consumed greedily @@ -1475,6 +1495,23 @@ The following recipes have a more mathematical flavor: True + >>> small_primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97] + >>> list(filter(is_prime, range(-100, 100))) == small_primes + True + >>> carmichael = {561, 1105, 1729, 2465, 2821, 6601, 8911} # https://oeis.org/A002997 + >>> any(map(is_prime, carmichael)) + False + >>> # https://www.wolframalpha.com/input?i=is+128884753939+prime + >>> is_prime(128_884_753_939) # large prime + True + >>> is_prime(999953 * 999983) # large semiprime + False + >>> is_prime(1_000_000_000_000_007) # factor() example + False + >>> is_prime(1_000_000_000_000_403) # factor() example + True + + >>> list(factor(99)) # Code example 1 [3, 3, 11] >>> list(factor(1_000_000_000_000_007)) # Code example 2 From 6fc643674983e27ec5cc312f2e83468050d1d364 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Dec 2024 08:58:22 +0000 Subject: [PATCH 096/427] gh-127572: Fix `test_structmembers` initialization (GH-127577) gh-127572: Fix `test_structmembers` initialization. The 'C' format code expects an `int` as a destination (not a `char`). This led to test failures on big-endian platforms like s390x. Use the 'c' format code, which expects a `char` as the destination (but requires a Python byte objects instead of a str). --- Lib/test/test_capi/test_structmembers.py | 2 +- Modules/_testcapi/structmember.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index ae9168fc39243f..f14ad9a9a5f512 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -39,7 +39,7 @@ def _make_test_object(cls): "hi", # T_STRING_INPLACE 12, # T_LONGLONG 13, # T_ULONGLONG - "c", # T_CHAR + b"c", # T_CHAR ) diff --git a/Modules/_testcapi/structmember.c b/Modules/_testcapi/structmember.c index c1861db18c4af2..ef30a5a9944e3c 100644 --- a/Modules/_testcapi/structmember.c +++ b/Modules/_testcapi/structmember.c @@ -60,7 +60,7 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "T_FLOAT", "T_DOUBLE", "T_STRING_INPLACE", "T_LONGLONG", "T_ULONGLONG", "T_CHAR", NULL}; - static const char fmt[] = "|bbBhHiIlknfds#LKC"; + static const char fmt[] = "|bbBhHiIlknfds#LKc"; test_structmembers *ob; const char *s = NULL; Py_ssize_t string_len = 0; From ad9d059eb10ef132edd73075fa6d8d96d95b8701 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 4 Dec 2024 13:01:46 +0300 Subject: [PATCH 097/427] gh-126524: Run `regen-unicodedata` as a part of our CI (#126682) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f8c468475470c..55effee0e1e393 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -120,7 +120,7 @@ jobs: - name: Build CPython run: | make -j4 regen-all - make regen-stdlib-module-names regen-sbom + make regen-stdlib-module-names regen-sbom regen-unicodedata - name: Check for changes run: | git add -u From bc0f2e945993747c8b1a6dd66cbe902fddd5758b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:13:52 +0100 Subject: [PATCH 098/427] gh-123378: Ensure results of `PyUnicode*Error_Get{Start,End}` are clamped (GH-123380) Co-authored-by: Sergey B Kirpichev --- Doc/c-api/exceptions.rst | 20 +- Doc/library/exceptions.rst | 6 + Lib/test/test_capi/test_exceptions.py | 150 +++++++++++ ...-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst | 6 + ...-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst | 6 + Modules/_testcapi/exceptions.c | 167 ++++++++++++ Objects/exceptions.c | 248 ++++++++++-------- 7 files changed, 492 insertions(+), 111 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst create mode 100644 Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index fc2336d120c259..c1f0bd750361d6 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -853,12 +853,23 @@ The following functions are used to create and modify Unicode exceptions from C. *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. + If the :attr:`UnicodeError.object` is an empty sequence, the resulting + *start* is ``0``. Otherwise, it is clipped to ``[0, len(object) - 1]``. + + .. seealso:: :attr:`UnicodeError.start` + .. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) - Set the *start* attribute of the given exception object to *start*. Return - ``0`` on success, ``-1`` on failure. + Set the *start* attribute of the given exception object to *start*. + Return ``0`` on success, ``-1`` on failure. + + .. note:: + + While passing a negative *start* does not raise an exception, + the corresponding getters will not consider it as a relative + offset. .. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) @@ -868,6 +879,9 @@ The following functions are used to create and modify Unicode exceptions from C. *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. + If the :attr:`UnicodeError.object` is an empty sequence, the resulting + *end* is ``0``. Otherwise, it is clipped to ``[1, len(object)]``. + .. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) @@ -875,6 +889,8 @@ The following functions are used to create and modify Unicode exceptions from C. Set the *end* attribute of the given exception object to *end*. Return ``0`` on success, ``-1`` on failure. + .. seealso:: :attr:`UnicodeError.end` + .. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index b5ba86f1b19223..f72b11e34c5c3d 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -644,10 +644,16 @@ The following exceptions are the exceptions that are usually raised. The first index of invalid data in :attr:`object`. + This value should not be negative as it is interpreted as an + absolute offset but this constraint is not enforced at runtime. + .. attribute:: end The index after the last invalid data in :attr:`object`. + This value should not be negative as it is interpreted as an + absolute offset but this constraint is not enforced at runtime. + .. exception:: UnicodeEncodeError diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index b22ddd8ad858d4..666e2f2ab09548 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -415,6 +415,156 @@ def test_err_formatunraisable(self): # CRASHES formatunraisable(NULL, NULL) +class TestUnicodeTranslateError(UnicodeTranslateError): + # UnicodeTranslateError takes 4 arguments instead of 5, + # so we just make a UnicodeTranslateError class that is + # compatible with the UnicodeError.__init__. + def __init__(self, encoding, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class TestUnicodeError(unittest.TestCase): + + def _check_no_crash(self, exc): + # ensure that the __str__() method does not crash + _ = str(exc) + + def test_unicode_encode_error_get_start(self): + get_start = _testcapi.unicode_encode_get_start + self._test_unicode_error_get_start('x', UnicodeEncodeError, get_start) + + def test_unicode_decode_error_get_start(self): + get_start = _testcapi.unicode_decode_get_start + self._test_unicode_error_get_start(b'x', UnicodeDecodeError, get_start) + + def test_unicode_translate_error_get_start(self): + get_start = _testcapi.unicode_translate_get_start + self._test_unicode_error_get_start('x', TestUnicodeTranslateError, get_start) + + def _test_unicode_error_get_start(self, literal, exc_type, get_start): + for obj_len, start, c_start in [ + # normal cases + (5, 0, 0), + (5, 1, 1), + (5, 2, 2), + # out of range start is clamped to max(0, obj_len - 1) + (0, 0, 0), + (0, 1, 0), + (0, 10, 0), + (5, 5, 4), + (5, 10, 4), + # negative values are allowed but clipped in the getter + (0, -1, 0), + (1, -1, 0), + (2, -1, 0), + (2, -2, 0), + ]: + obj = literal * obj_len + with self.subTest(obj, exc_type=exc_type, start=start): + exc = exc_type('utf-8', obj, start, obj_len, 'reason') + self.assertEqual(get_start(exc), c_start) + self._check_no_crash(exc) + + def test_unicode_encode_error_set_start(self): + set_start = _testcapi.unicode_encode_set_start + self._test_unicode_error_set_start('x', UnicodeEncodeError, set_start) + + def test_unicode_decode_error_set_start(self): + set_start = _testcapi.unicode_decode_set_start + self._test_unicode_error_set_start(b'x', UnicodeDecodeError, set_start) + + def test_unicode_translate_error_set_start(self): + set_start = _testcapi.unicode_translate_set_start + self._test_unicode_error_set_start('x', TestUnicodeTranslateError, set_start) + + def _test_unicode_error_set_start(self, literal, exc_type, set_start): + obj_len = 5 + obj = literal * obj_len + for new_start in range(-2 * obj_len, 2 * obj_len): + with self.subTest('C-API', obj=obj, exc_type=exc_type, new_start=new_start): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the C API setter + set_start(exc, new_start) + self.assertEqual(exc.start, new_start) + self._check_no_crash(exc) + + with self.subTest('Py-API', obj=obj, exc_type=exc_type, new_start=new_start): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the attribute setter + exc.start = new_start + self.assertEqual(exc.start, new_start) + self._check_no_crash(exc) + + def test_unicode_encode_error_get_end(self): + get_end = _testcapi.unicode_encode_get_end + self._test_unicode_error_get_end('x', UnicodeEncodeError, get_end) + + def test_unicode_decode_error_get_end(self): + get_end = _testcapi.unicode_decode_get_end + self._test_unicode_error_get_end(b'x', UnicodeDecodeError, get_end) + + def test_unicode_translate_error_get_end(self): + get_end = _testcapi.unicode_translate_get_end + self._test_unicode_error_get_end('x', TestUnicodeTranslateError, get_end) + + def _test_unicode_error_get_end(self, literal, exc_type, get_end): + for obj_len, end, c_end in [ + # normal cases + (5, 0, 1), + (5, 1, 1), + (5, 2, 2), + # out-of-range clipped in [MIN(1, OBJLEN), MAX(MIN(1, OBJLEN), OBJLEN)] + (0, 0, 0), + (0, 1, 0), + (0, 10, 0), + (1, 1, 1), + (1, 2, 1), + (5, 5, 5), + (5, 5, 5), + (5, 10, 5), + # negative values are allowed but clipped in the getter + (0, -1, 0), + (1, -1, 1), + (2, -1, 1), + (2, -2, 1), + ]: + obj = literal * obj_len + with self.subTest(obj, exc_type=exc_type, end=end): + exc = exc_type('utf-8', obj, 0, end, 'reason') + self.assertEqual(get_end(exc), c_end) + self._check_no_crash(exc) + + def test_unicode_encode_error_set_end(self): + set_end = _testcapi.unicode_encode_set_end + self._test_unicode_error_set_end('x', UnicodeEncodeError, set_end) + + def test_unicode_decode_error_set_end(self): + set_end = _testcapi.unicode_decode_set_end + self._test_unicode_error_set_end(b'x', UnicodeDecodeError, set_end) + + def test_unicode_translate_error_set_end(self): + set_end = _testcapi.unicode_translate_set_end + self._test_unicode_error_set_end('x', TestUnicodeTranslateError, set_end) + + def _test_unicode_error_set_end(self, literal, exc_type, set_end): + obj_len = 5 + obj = literal * obj_len + for new_end in range(-2 * obj_len, 2 * obj_len): + with self.subTest('C-API', obj=obj, exc_type=exc_type, new_end=new_end): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the C API setter + set_end(exc, new_end) + self.assertEqual(exc.end, new_end) + self._check_no_crash(exc) + + with self.subTest('Py-API', obj=obj, exc_type=exc_type, new_end=new_end): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the attribute setter + exc.end = new_end + self.assertEqual(exc.end, new_end) + self._check_no_crash(exc) + + class Test_PyUnstable_Exc_PrepReraiseStar(ExceptionIsLikeMixin, unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst b/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst new file mode 100644 index 00000000000000..2cfb8b8a1e245a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst @@ -0,0 +1,6 @@ +Ensure that the value of :attr:`UnicodeEncodeError.start ` +retrieved by :c:func:`PyUnicodeEncodeError_GetStart` lie in +``[0, max(0, objlen - 1)]`` where *objlen* is the length of +:attr:`UnicodeEncodeError.object `. Similar +arguments apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` +and their corresponding C interface. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst b/Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst new file mode 100644 index 00000000000000..107751579c4d91 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst @@ -0,0 +1,6 @@ +Ensure that the value of :attr:`UnicodeEncodeError.end ` +retrieved by :c:func:`PyUnicodeEncodeError_GetEnd` lies in ``[min(1, objlen), +max(min(1, objlen), objlen)]`` where *objlen* is the length of +:attr:`UnicodeEncodeError.object `. Similar arguments +apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` and their +corresponding C interface. Patch by Bénédikt Tran. diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index 316ef0e7ad7e55..e92d9670e7c792 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -359,6 +359,161 @@ _testcapi_unstable_exc_prep_reraise_star_impl(PyObject *module, return PyUnstable_Exc_PrepReraiseStar(orig, excs); } +/* Test PyUnicodeEncodeError_GetStart */ +static PyObject * +unicode_encode_get_start(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t start; + if (PyUnicodeEncodeError_GetStart(arg, &start) < 0) { + return NULL; + } + RETURN_SIZE(start); +} + +/* Test PyUnicodeDecodeError_GetStart */ +static PyObject * +unicode_decode_get_start(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t start; + if (PyUnicodeDecodeError_GetStart(arg, &start) < 0) { + return NULL; + } + RETURN_SIZE(start); +} + +/* Test PyUnicodeTranslateError_GetStart */ +static PyObject * +unicode_translate_get_start(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t start; + if (PyUnicodeTranslateError_GetStart(arg, &start) < 0) { + return NULL; + } + RETURN_SIZE(start); +} + +/* Test PyUnicodeEncodeError_SetStart */ +static PyObject * +unicode_encode_set_start(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t start; + if (PyArg_ParseTuple(args, "On", &exc, &start) < 0) { + return NULL; + } + if (PyUnicodeEncodeError_SetStart(exc, start) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeDecodeError_SetStart */ +static PyObject * +unicode_decode_set_start(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t start; + if (PyArg_ParseTuple(args, "On", &exc, &start) < 0) { + return NULL; + } + if (PyUnicodeDecodeError_SetStart(exc, start) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeTranslateError_SetStart */ +static PyObject * +unicode_translate_set_start(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t start; + if (PyArg_ParseTuple(args, "On", &exc, &start) < 0) { + return NULL; + } + if (PyUnicodeTranslateError_SetStart(exc, start) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeEncodeError_GetEnd */ +static PyObject * +unicode_encode_get_end(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t end; + if (PyUnicodeEncodeError_GetEnd(arg, &end) < 0) { + return NULL; + } + RETURN_SIZE(end); +} + +/* Test PyUnicodeDecodeError_GetEnd */ +static PyObject * +unicode_decode_get_end(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t end; + if (PyUnicodeDecodeError_GetEnd(arg, &end) < 0) { + return NULL; + } + RETURN_SIZE(end); +} + +/* Test PyUnicodeTranslateError_GetEnd */ +static PyObject * +unicode_translate_get_end(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t end; + if (PyUnicodeTranslateError_GetEnd(arg, &end) < 0) { + return NULL; + } + RETURN_SIZE(end); +} + +/* Test PyUnicodeEncodeError_SetEnd */ +static PyObject * +unicode_encode_set_end(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t end; + if (PyArg_ParseTuple(args, "On", &exc, &end) < 0) { + return NULL; + } + if (PyUnicodeEncodeError_SetEnd(exc, end) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeDecodeError_SetEnd */ +static PyObject * +unicode_decode_set_end(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t end; + if (PyArg_ParseTuple(args, "On", &exc, &end) < 0) { + return NULL; + } + if (PyUnicodeDecodeError_SetEnd(exc, end) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeTranslateError_SetEnd */ +static PyObject * +unicode_translate_set_end(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t end; + if (PyArg_ParseTuple(args, "On", &exc, &end) < 0) { + return NULL; + } + if (PyUnicodeTranslateError_SetEnd(exc, end) < 0) { + return NULL; + } + Py_RETURN_NONE; +} /* * Define the PyRecurdingInfinitelyError_Type @@ -403,6 +558,18 @@ static PyMethodDef test_methods[] = { _TESTCAPI_SET_EXCEPTION_METHODDEF _TESTCAPI_TRACEBACK_PRINT_METHODDEF _TESTCAPI_UNSTABLE_EXC_PREP_RERAISE_STAR_METHODDEF + {"unicode_encode_get_start", unicode_encode_get_start, METH_O}, + {"unicode_decode_get_start", unicode_decode_get_start, METH_O}, + {"unicode_translate_get_start", unicode_translate_get_start, METH_O}, + {"unicode_encode_set_start", unicode_encode_set_start, METH_VARARGS}, + {"unicode_decode_set_start", unicode_decode_set_start, METH_VARARGS}, + {"unicode_translate_set_start", unicode_translate_set_start, METH_VARARGS}, + {"unicode_encode_get_end", unicode_encode_get_end, METH_O}, + {"unicode_decode_get_end", unicode_decode_get_end, METH_O}, + {"unicode_translate_get_end", unicode_translate_get_end, METH_O}, + {"unicode_encode_set_end", unicode_encode_set_end, METH_VARARGS}, + {"unicode_decode_set_end", unicode_decode_set_end, METH_VARARGS}, + {"unicode_translate_set_end", unicode_translate_set_end, METH_VARARGS}, {NULL}, }; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 6fbe0f197eaebf..124b591ee3a13f 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2708,6 +2708,46 @@ set_unicodefromstring(PyObject **attr, const char *value) return 0; } +/* + * Adjust the (inclusive) 'start' value of a UnicodeError object. + * + * The 'start' can be negative or not, but when adjusting the value, + * we clip it in [0, max(0, objlen - 1)] but do not intepret it as + * a relative offset. + */ +static inline Py_ssize_t +unicode_error_adjust_start(Py_ssize_t start, Py_ssize_t objlen) +{ + assert(objlen >= 0); + if (start < 0) { + start = 0; + } + if (start >= objlen) { + start = objlen == 0 ? 0 : objlen - 1; + } + return start; +} + +/* + * Adjust the (exclusive) 'end' value of a UnicodeError object. + * + * The 'end' can be negative or not, but when adjusting the value, + * we clip it in [min(1, objlen), max(min(1, objlen), objlen)] but + * do not intepret it as a relative offset. + */ +static inline Py_ssize_t +unicode_error_adjust_end(Py_ssize_t end, Py_ssize_t objlen) +{ + assert(objlen >= 0); + if (end < 1) { + end = 1; + } + if (end > objlen) { + end = objlen; + } + return end; +} + PyObject * PyUnicodeEncodeError_GetEncoding(PyObject *exc) { @@ -2739,38 +2779,31 @@ PyUnicodeTranslateError_GetObject(PyObject *exc) } int -PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) +PyUnicodeEncodeError_GetStart(PyObject *self, Py_ssize_t *start) { - Py_ssize_t size; - PyObject *obj = get_unicode(((PyUnicodeErrorObject *)exc)->object, - "object"); - if (!obj) + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyObject *obj = get_unicode(exc->object, "object"); + if (obj == NULL) { return -1; - *start = ((PyUnicodeErrorObject *)exc)->start; - size = PyUnicode_GET_LENGTH(obj); - if (*start<0) - *start = 0; /*XXX check for values <0*/ - if (*start>=size) - *start = size-1; + } + Py_ssize_t size = PyUnicode_GET_LENGTH(obj); Py_DECREF(obj); + *start = unicode_error_adjust_start(exc->start, size); return 0; } int -PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) +PyUnicodeDecodeError_GetStart(PyObject *self, Py_ssize_t *start) { - Py_ssize_t size; - PyObject *obj = get_string(((PyUnicodeErrorObject *)exc)->object, "object"); - if (!obj) + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyObject *obj = get_string(exc->object, "object"); + if (obj == NULL) { return -1; - size = PyBytes_GET_SIZE(obj); - *start = ((PyUnicodeErrorObject *)exc)->start; - if (*start<0) - *start = 0; - if (*start>=size) - *start = size-1; + } + Py_ssize_t size = PyBytes_GET_SIZE(obj); Py_DECREF(obj); + *start = unicode_error_adjust_start(exc->start, size); return 0; } @@ -2782,63 +2815,61 @@ PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) } +static inline int +unicode_error_set_start_impl(PyObject *self, Py_ssize_t start) +{ + ((PyUnicodeErrorObject *)self)->start = start; + return 0; +} + + int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) { - ((PyUnicodeErrorObject *)exc)->start = start; - return 0; + return unicode_error_set_start_impl(exc, start); } int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) { - ((PyUnicodeErrorObject *)exc)->start = start; - return 0; + return unicode_error_set_start_impl(exc, start); } int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) { - ((PyUnicodeErrorObject *)exc)->start = start; - return 0; + return unicode_error_set_start_impl(exc, start); } int -PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) +PyUnicodeEncodeError_GetEnd(PyObject *self, Py_ssize_t *end) { - Py_ssize_t size; - PyObject *obj = get_unicode(((PyUnicodeErrorObject *)exc)->object, - "object"); - if (!obj) + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyObject *obj = get_unicode(exc->object, "object"); + if (obj == NULL) { return -1; - *end = ((PyUnicodeErrorObject *)exc)->end; - size = PyUnicode_GET_LENGTH(obj); - if (*end<1) - *end = 1; - if (*end>size) - *end = size; + } + Py_ssize_t size = PyUnicode_GET_LENGTH(obj); Py_DECREF(obj); + *end = unicode_error_adjust_end(exc->end, size); return 0; } int -PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) +PyUnicodeDecodeError_GetEnd(PyObject *self, Py_ssize_t *end) { - Py_ssize_t size; - PyObject *obj = get_string(((PyUnicodeErrorObject *)exc)->object, "object"); - if (!obj) + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyObject *obj = get_string(exc->object, "object"); + if (obj == NULL) { return -1; - size = PyBytes_GET_SIZE(obj); - *end = ((PyUnicodeErrorObject *)exc)->end; - if (*end<1) - *end = 1; - if (*end>size) - *end = size; + } + Py_ssize_t size = PyBytes_GET_SIZE(obj); Py_DECREF(obj); + *end = unicode_error_adjust_end(exc->end, size); return 0; } @@ -2850,27 +2881,32 @@ PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) } -int -PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) +static inline int +unicode_error_set_end_impl(PyObject *exc, Py_ssize_t end) { ((PyUnicodeErrorObject *)exc)->end = end; return 0; } +int +PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) +{ + return unicode_error_set_end_impl(exc, end); +} + + int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) { - ((PyUnicodeErrorObject *)exc)->end = end; - return 0; + return unicode_error_set_end_impl(exc, end); } int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) { - ((PyUnicodeErrorObject *)exc)->end = end; - return 0; + return unicode_error_set_end_impl(exc, end); } PyObject * @@ -2966,28 +3002,25 @@ static PyMemberDef UnicodeError_members[] = { static int UnicodeEncodeError_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyUnicodeErrorObject *err; - - if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) { return -1; + } - err = (PyUnicodeErrorObject *)self; - - Py_CLEAR(err->encoding); - Py_CLEAR(err->object); - Py_CLEAR(err->reason); + PyObject *encoding = NULL, *object = NULL, *reason = NULL; // borrowed + Py_ssize_t start = -1, end = -1; if (!PyArg_ParseTuple(args, "UUnnU", - &err->encoding, &err->object, - &err->start, &err->end, &err->reason)) { - err->encoding = err->object = err->reason = NULL; + &encoding, &object, &start, &end, &reason)) + { return -1; } - Py_INCREF(err->encoding); - Py_INCREF(err->object); - Py_INCREF(err->reason); - + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + Py_XSETREF(exc->encoding, Py_NewRef(encoding)); + Py_XSETREF(exc->object, Py_NewRef(object)); + exc->start = start; + exc->end = end; + Py_XSETREF(exc->reason, Py_NewRef(reason)); return 0; } @@ -3073,44 +3106,42 @@ PyObject *PyExc_UnicodeEncodeError = (PyObject *)&_PyExc_UnicodeEncodeError; static int UnicodeDecodeError_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyUnicodeErrorObject *ude; - - if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) { return -1; + } - ude = (PyUnicodeErrorObject *)self; - - Py_CLEAR(ude->encoding); - Py_CLEAR(ude->object); - Py_CLEAR(ude->reason); + PyObject *encoding = NULL, *object = NULL, *reason = NULL; // borrowed + Py_ssize_t start = -1, end = -1; if (!PyArg_ParseTuple(args, "UOnnU", - &ude->encoding, &ude->object, - &ude->start, &ude->end, &ude->reason)) { - ude->encoding = ude->object = ude->reason = NULL; - return -1; + &encoding, &object, &start, &end, &reason)) + { + return -1; } - Py_INCREF(ude->encoding); - Py_INCREF(ude->object); - Py_INCREF(ude->reason); - - if (!PyBytes_Check(ude->object)) { + if (PyBytes_Check(object)) { + Py_INCREF(object); // make 'object' a strong reference + } + else { Py_buffer view; - if (PyObject_GetBuffer(ude->object, &view, PyBUF_SIMPLE) != 0) - goto error; - Py_XSETREF(ude->object, PyBytes_FromStringAndSize(view.buf, view.len)); + if (PyObject_GetBuffer(object, &view, PyBUF_SIMPLE) != 0) { + return -1; + } + // 'object' is borrowed, so we can re-use the variable + object = PyBytes_FromStringAndSize(view.buf, view.len); PyBuffer_Release(&view); - if (!ude->object) - goto error; + if (object == NULL) { + return -1; + } } - return 0; -error: - Py_CLEAR(ude->encoding); - Py_CLEAR(ude->object); - Py_CLEAR(ude->reason); - return -1; + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + Py_XSETREF(exc->encoding, Py_NewRef(encoding)); + Py_XSETREF(exc->object, object /* already a strong reference */); + exc->start = start; + exc->end = end; + Py_XSETREF(exc->reason, Py_NewRef(reason)); + return 0; } static PyObject * @@ -3192,25 +3223,24 @@ PyUnicodeDecodeError_Create( */ static int -UnicodeTranslateError_init(PyUnicodeErrorObject *self, PyObject *args, - PyObject *kwds) +UnicodeTranslateError_init(PyObject *self, PyObject *args, PyObject *kwds) { - if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) { return -1; + } - Py_CLEAR(self->object); - Py_CLEAR(self->reason); + PyObject *object = NULL, *reason = NULL; // borrowed + Py_ssize_t start = -1, end = -1; - if (!PyArg_ParseTuple(args, "UnnU", - &self->object, - &self->start, &self->end, &self->reason)) { - self->object = self->reason = NULL; + if (!PyArg_ParseTuple(args, "UnnU", &object, &start, &end, &reason)) { return -1; } - Py_INCREF(self->object); - Py_INCREF(self->reason); - + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + Py_XSETREF(exc->object, Py_NewRef(object)); + exc->start = start; + exc->end = end; + Py_XSETREF(exc->reason, Py_NewRef(reason)); return 0; } From 6bc3e830a518112a4e242217807681e3908602f4 Mon Sep 17 00:00:00 2001 From: "RUANG (James Roy)" Date: Wed, 4 Dec 2024 21:30:38 +0800 Subject: [PATCH 099/427] gh-127481: Add `EPOLLWAKEUP` to the `select` module (GH-127482) --- Doc/library/select.rst | 6 ++++++ .../Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst | 1 + Modules/selectmodule.c | 4 ++++ 3 files changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst diff --git a/Doc/library/select.rst b/Doc/library/select.rst index f23a249f44b485..4fcff9198944a8 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -317,11 +317,17 @@ Edge and Level Trigger Polling (epoll) Objects +-------------------------+-----------------------------------------------+ | :const:`EPOLLMSG` | Ignored. | +-------------------------+-----------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+-----------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 or later. + .. versionadded:: next + :const:`EPOLLWAKEUP` was added. It's only supported by Linux Kernel 3.5 + or later. + .. method:: epoll.close() Close the control file descriptor of the epoll object. diff --git a/Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst b/Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst new file mode 100644 index 00000000000000..8ada0b57ddc257 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst @@ -0,0 +1 @@ +Add the ``EPOLLWAKEUP`` constant to the :mod:`select` module. diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 6ced71cca73178..e14e114a6dafd0 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2715,6 +2715,10 @@ _select_exec(PyObject *m) #ifdef EPOLLMSG ADD_INT(EPOLLMSG); #endif +#ifdef EPOLLWAKEUP + /* Kernel 3.5+ */ + ADD_INT(EPOLLWAKEUP); +#endif #ifdef EPOLL_CLOEXEC ADD_INT(EPOLL_CLOEXEC); From 51cfa569e379f84b3418db0971a71b1ef575a42b Mon Sep 17 00:00:00 2001 From: Beomsoo Kim Date: Thu, 5 Dec 2024 03:30:51 +0900 Subject: [PATCH 100/427] =?UTF-8?q?gh-127552:=20Remove=20comment=20questio?= =?UTF-8?q?ning=204-digit=20restriction=20for=20=E2=80=98Y=E2=80=99=20in?= =?UTF-8?q?=20datetime.strptime=20patterns=20(#127590)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code has required 4 digits for the year since its inclusion in the stdlib in 2002 (over 22 years ago as of this commit). --- Lib/_strptime.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 5f4d2475c0169b..e6e23596db6f99 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -301,8 +301,6 @@ def __init__(self, locale_time=None): 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", # W is set below by using 'U' 'y': r"(?P\d\d)", - #XXX: Does 'Y' need to worry about having less or more than - # 4 digits? 'Y': r"(?P\d\d\d\d)", 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), From e51da64ac3bc6cd45339864db32d05115af39ead Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Dec 2024 19:12:15 +0000 Subject: [PATCH 101/427] gh-127536: Add missing locks in listobject.c (GH-127580) We were missing locks around some list operations in the free threading build. --- ...-12-03-21-07-06.gh-issue-127536.3jMMrT.rst | 2 + Objects/listobject.c | 50 +++++++++++++++---- 2 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst new file mode 100644 index 00000000000000..6e2b87fe38146b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst @@ -0,0 +1,2 @@ +Add missing locks around some list assignment operations in the free +threading build. diff --git a/Objects/listobject.c b/Objects/listobject.c index 8abe9e8933420b..3832295600a0ab 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_dict.h" // _PyDictViewObject #include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_pyatomic_ft_wrappers.h" @@ -72,6 +73,11 @@ static void ensure_shared_on_resize(PyListObject *self) { #ifdef Py_GIL_DISABLED + // We can't use _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED here because + // the `CALL_LIST_APPEND` bytecode handler may lock the list without + // a critical section. + assert(Py_REFCNT(self) == 1 || PyMutex_IsLocked(&_PyObject_CAST(self)->ob_mutex)); + // Ensure that the list array is freed using QSBR if we are not the // owning thread. if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && @@ -957,10 +963,12 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) Py_ssize_t n = PyList_GET_SIZE(a); PyObject *copy = list_slice_lock_held(a, 0, n); if (copy == NULL) { - return -1; + ret = -1; + } + else { + ret = list_ass_slice_lock_held(a, ilow, ihigh, copy); + Py_DECREF(copy); } - ret = list_ass_slice_lock_held(a, ilow, ihigh, copy); - Py_DECREF(copy); Py_END_CRITICAL_SECTION(); } else if (v != NULL && PyList_CheckExact(v)) { @@ -1437,7 +1445,9 @@ PyList_Clear(PyObject *self) PyErr_BadInternalCall(); return -1; } + Py_BEGIN_CRITICAL_SECTION(self); list_clear((PyListObject*)self); + Py_END_CRITICAL_SECTION(); return 0; } @@ -3410,7 +3420,9 @@ list___init___impl(PyListObject *self, PyObject *iterable) /* Empty previous contents */ if (self->ob_item != NULL) { + Py_BEGIN_CRITICAL_SECTION(self); list_clear(self); + Py_END_CRITICAL_SECTION(); } if (iterable != NULL) { if (_list_extend(self, iterable) < 0) { @@ -3583,8 +3595,10 @@ adjust_slice_indexes(PyListObject *lst, } static int -list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) +list_ass_subscript_lock_held(PyObject *_self, PyObject *item, PyObject *value) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(_self); + PyListObject *self = (PyListObject *)_self; if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -3592,7 +3606,7 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) return -1; if (i < 0) i += PyList_GET_SIZE(self); - return list_ass_item((PyObject *)self, i, value); + return list_ass_item_lock_held(self, i, value); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step; @@ -3612,7 +3626,7 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) step); if (step == 1) - return list_ass_slice(self, start, stop, value); + return list_ass_slice_lock_held(self, start, stop, value); if (slicelength <= 0) return 0; @@ -3678,10 +3692,8 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) /* protect against a[::-1] = a */ if (self == (PyListObject*)value) { - Py_BEGIN_CRITICAL_SECTION(value); - seq = list_slice_lock_held((PyListObject*)value, 0, + seq = list_slice_lock_held((PyListObject *)value, 0, Py_SIZE(value)); - Py_END_CRITICAL_SECTION(); } else { seq = PySequence_Fast(value, @@ -3695,7 +3707,7 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) step); if (step == 1) { - int res = list_ass_slice(self, start, stop, seq); + int res = list_ass_slice_lock_held(self, start, stop, seq); Py_DECREF(seq); return res; } @@ -3751,6 +3763,24 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) } } +static int +list_ass_subscript(PyObject *self, PyObject *item, PyObject *value) +{ + int res; +#ifdef Py_GIL_DISABLED + if (PySlice_Check(item) && value != NULL && PyList_CheckExact(value)) { + Py_BEGIN_CRITICAL_SECTION2(self, value); + res = list_ass_subscript_lock_held(self, item, value); + Py_END_CRITICAL_SECTION2(); + return res; + } +#endif + Py_BEGIN_CRITICAL_SECTION(self); + res = list_ass_subscript_lock_held(self, item, value); + Py_END_CRITICAL_SECTION(); + return res; +} + static PyMappingMethods list_as_mapping = { list_length, list_subscript, From 7c5a6f67c726608a05a640e76fc62cfbae986a03 Mon Sep 17 00:00:00 2001 From: Diego Russo Date: Wed, 4 Dec 2024 22:12:06 +0000 Subject: [PATCH 102/427] Enable native AArch64 Ubuntu CI jobs (#127584) Co-authored-by: Brandt Bucher --- .github/workflows/build.yml | 9 +++ .github/workflows/jit.yml | 31 ++------- .github/workflows/reusable-ubuntu.yml | 16 ++--- Tools/jit/ignore-tests-emulated-linux.txt | 85 ----------------------- 4 files changed, 24 insertions(+), 117 deletions(-) delete mode 100644 Tools/jit/ignore-tests-emulated-linux.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 55effee0e1e393..9b2f19fd6bcf54 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -235,10 +235,19 @@ jobs: free-threading: - false - true + os: + - ubuntu-24.04 + - ubuntu-24.04-aarch64 + is-fork: # only used for the exclusion trick + - ${{ github.repository_owner != 'python' }} + exclude: + - os: ubuntu-24.04-aarch64 + is-fork: true uses: ./.github/workflows/reusable-ubuntu.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} free-threading: ${{ matrix.free-threading }} + os: ${{ matrix.os }} build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 4ef543d7369734..ee30cf5786d55b 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -28,7 +28,7 @@ concurrency: jobs: interpreter: name: Interpreter (Debug) - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -78,10 +78,11 @@ jobs: runner: macos-14 - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 - runner: ubuntu-22.04 + runner: ubuntu-24.04 - target: aarch64-unknown-linux-gnu/gcc architecture: aarch64 - runner: ubuntu-22.04 + # Forks don't have access to our paid AArch64 runners. These jobs are skipped below: + runner: ${{ github.repository_owner == 'python' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -118,7 +119,8 @@ jobs: ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - name: Native Linux - if: runner.os == 'Linux' && matrix.architecture == 'x86_64' + # Forks don't have access to our paid AArch64 runners. Skip those: + if: runner.os == 'Linux' && (matrix.architecture == 'x86_64' || github.repository_owner == 'python') run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" @@ -126,29 +128,10 @@ jobs: make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Emulated Linux - if: runner.os == 'Linux' && matrix.architecture != 'x86_64' - # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux. - run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --prefix="$(pwd)/../build" - make install --jobs 4 - make clean --jobs 4 - export HOST=${{ matrix.architecture }}-linux-gnu - sudo apt install --yes "gcc-$HOST" qemu-user - export QEMU_LD_PREFIX="/usr/$HOST" - CC="$HOST-gcc" \ - CPP="$HOST-gcc --preprocess" \ - HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes - make all --jobs 4 - ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - jit-with-disabled-gil: name: Free-Threaded (Debug) needs: interpreter - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: llvm: diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 642354f8b4f61b..2869202c7910c9 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -11,16 +11,16 @@ on: required: false type: boolean default: false + os: + description: OS to run the job + required: true + type: string jobs: build_ubuntu_reusable: - name: 'build and test' + name: build and test (${{ inputs.os }}) timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04, ubuntu-24.04-aarch64] + runs-on: ${{ inputs.os }} env: FORCE_COLOR: 1 OPENSSL_VER: 3.0.15 @@ -42,7 +42,7 @@ jobs: uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux @@ -84,7 +84,7 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j - name: Build CPython out-of-tree (for compiler warning check) - if: ${{ !inputs.free-threading}} + if: ${{ !inputs.free-threading }} working-directory: ${{ env.CPYTHON_BUILDDIR }} run: set -o pipefail; make -j --output-sync 2>&1 | tee compiler_output_ubuntu.txt - name: Display build info diff --git a/Tools/jit/ignore-tests-emulated-linux.txt b/Tools/jit/ignore-tests-emulated-linux.txt deleted file mode 100644 index 080a569574470c..00000000000000 --- a/Tools/jit/ignore-tests-emulated-linux.txt +++ /dev/null @@ -1,85 +0,0 @@ -test_multiprocessing_fork -test_strftime_y2k -test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_run -test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_subprocess -test.test_asyncio.test_unix_events.TestFork.test_fork_signal_handling -test.test_cmd_line.CmdLineTest.test_no_std_streams -test.test_cmd_line.CmdLineTest.test_no_stdin -test.test_concurrent_futures.test_init.ProcessPoolForkFailingInitializerTest.test_initializer -test.test_concurrent_futures.test_process_pool.ProcessPoolForkProcessPoolExecutorTest.test_ressources_gced_in_workers -test.test_external_inspection.TestGetStackTrace.test_remote_stack_trace -test.test_external_inspection.TestGetStackTrace.test_self_trace -test.test_faulthandler.FaultHandlerTests.test_enable_fd -test.test_faulthandler.FaultHandlerTests.test_enable_file -test.test_init.ProcessPoolForkFailingInitializerTest.test_initializer -test.test_logging.ConfigDictTest.test_111615 -test.test_logging.ConfigDictTest.test_config_queue_handler -test.test_logging.ConfigDictTest.test_multiprocessing_queues -test.test_logging.ConfigDictTest.test_config_queue_handler_multiprocessing_context -test.test_os.ForkTests.test_fork_warns_when_non_python_thread_exists -test.test_os.TimerfdTests.test_timerfd_initval -test.test_os.TimerfdTests.test_timerfd_interval -test.test_os.TimerfdTests.test_timerfd_TFD_TIMER_ABSTIME -test.test_pathlib.PathSubclassTest.test_is_mount_root -test.test_pathlib.PathTest.test_is_mount_root -test.test_pathlib.PosixPathTest.test_is_mount_root -test.test_pathlib.test_pathlib.PathSubclassTest.test_is_mount_root -test.test_pathlib.test_pathlib.PathTest.test_is_mount_root -test.test_pathlib.test_pathlib.PosixPathTest.test_is_mount_root -test.test_posix.TestPosixSpawn.test_close_file -test.test_posix.TestPosixSpawnP.test_close_file -test.test_posixpath.PosixPathTest.test_ismount -test.test_signal.StressTest.test_stress_modifying_handlers -test.test_socket.BasicCANTest.testFilter -test.test_socket.BasicCANTest.testLoopback -test.test_socket.LinuxKernelCryptoAPI.test_aead_aes_gcm -test.test_socket.LinuxKernelCryptoAPI.test_aes_cbc -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 -test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0 -test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Minus1 -test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Plus1 -test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen1 -test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen2Minus1 -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 -test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0 -test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Minus1 -test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Plus1 -test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen1 -test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen2Minus1 -test.test_subprocess.POSIXProcessTestCase.test_exception_bad_args_0 -test.test_subprocess.POSIXProcessTestCase.test_exception_bad_executable -test.test_subprocess.POSIXProcessTestCase.test_vfork_used_when_expected -test.test_subprocess.ProcessTestCase.test_cwd_with_relative_arg -test.test_subprocess.ProcessTestCase.test_cwd_with_relative_executable -test.test_subprocess.ProcessTestCase.test_empty_env -test.test_subprocess.ProcessTestCase.test_file_not_found_includes_filename -test.test_subprocess.ProcessTestCase.test_one_environment_variable -test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_arg -test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_executable -test.test_subprocess.ProcessTestCaseNoPoll.test_empty_env -test.test_subprocess.ProcessTestCaseNoPoll.test_file_not_found_includes_filename -test.test_subprocess.ProcessTestCaseNoPoll.test_one_environment_variable -test.test_venv.BasicTest.test_zippath_from_non_installed_posix From 94b8f8b40943bf38cf5c454773a3fb8f4ff71e01 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 4 Dec 2024 15:01:28 -0800 Subject: [PATCH 103/427] GH-126795: Increase the JIT side-exit threshold from 64 to 4096 (GH-127155) --- Include/internal/pycore_backoff.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 3e02728522828e..b5e33fa8b7abc0 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -115,10 +115,9 @@ initial_jump_backoff_counter(void) /* Initial exit temperature. * Must be larger than ADAPTIVE_COOLDOWN_VALUE, * otherwise when a side exit warms up we may construct - * a new trace before the Tier 1 code has properly re-specialized. - * Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */ -#define SIDE_EXIT_INITIAL_VALUE 63 -#define SIDE_EXIT_INITIAL_BACKOFF 6 + * a new trace before the Tier 1 code has properly re-specialized. */ +#define SIDE_EXIT_INITIAL_VALUE 4095 +#define SIDE_EXIT_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter initial_temperature_backoff_counter(void) From 2f1cee8477e22bfc36a704310e4c0f409357e7e9 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 5 Dec 2024 01:25:06 +0100 Subject: [PATCH 104/427] gh-127111: Apply prettier formatter to Emscripten web example (#127551) Cleaned up formatting (and a stray closing tag) of the web example HTML and JS. --- Tools/wasm/emscripten/web_example/python.html | 782 ++++++++++-------- .../emscripten/web_example/python.worker.mjs | 175 ++-- 2 files changed, 511 insertions(+), 446 deletions(-) diff --git a/Tools/wasm/emscripten/web_example/python.html b/Tools/wasm/emscripten/web_example/python.html index fae1e9ad4e8acb..078f86eb764419 100644 --- a/Tools/wasm/emscripten/web_example/python.html +++ b/Tools/wasm/emscripten/web_example/python.html @@ -1,373 +1,433 @@ - + - - - - - - - wasm-python terminal - - - - - - -

Simple REPL for Python WASM

- -
- - - - -
-
-
- The simple REPL provides a limited Python experience in the browser. - - Tools/wasm/README.md contains a list of known limitations and - issues. Networking, subprocesses, and threading are not available. -
- + +
+ + + + +
+
+
+ The simple REPL provides a limited Python experience in the browser. + + Tools/wasm/README.md + + contains a list of known limitations and issues. Networking, + subprocesses, and threading are not available. +
+ diff --git a/Tools/wasm/emscripten/web_example/python.worker.mjs b/Tools/wasm/emscripten/web_example/python.worker.mjs index 42c2e1e08af24b..8043e419966743 100644 --- a/Tools/wasm/emscripten/web_example/python.worker.mjs +++ b/Tools/wasm/emscripten/web_example/python.worker.mjs @@ -1,104 +1,109 @@ import createEmscriptenModule from "./python.mjs"; class StdinBuffer { - constructor() { - this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT) - this.buffer = new Int32Array(this.sab) - this.readIndex = 1; - this.numberOfCharacters = 0; - this.sentNull = true - } + constructor() { + this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT); + this.buffer = new Int32Array(this.sab); + this.readIndex = 1; + this.numberOfCharacters = 0; + this.sentNull = true; + } - prompt() { - this.readIndex = 1 - Atomics.store(this.buffer, 0, -1) - postMessage({ - type: 'stdin', - buffer: this.sab - }) - Atomics.wait(this.buffer, 0, -1) - this.numberOfCharacters = this.buffer[0] - } + prompt() { + this.readIndex = 1; + Atomics.store(this.buffer, 0, -1); + postMessage({ + type: "stdin", + buffer: this.sab, + }); + Atomics.wait(this.buffer, 0, -1); + this.numberOfCharacters = this.buffer[0]; + } - stdin = () => { - while (this.numberOfCharacters + 1 === this.readIndex) { - if (!this.sentNull) { - // Must return null once to indicate we're done for now. - this.sentNull = true - return null - } - this.sentNull = false - // Prompt will reset this.readIndex to 1 - this.prompt() - } - const char = this.buffer[this.readIndex] - this.readIndex += 1 - return char + stdin = () => { + while (this.numberOfCharacters + 1 === this.readIndex) { + if (!this.sentNull) { + // Must return null once to indicate we're done for now. + this.sentNull = true; + return null; + } + this.sentNull = false; + // Prompt will reset this.readIndex to 1 + this.prompt(); } + const char = this.buffer[this.readIndex]; + this.readIndex += 1; + return char; + }; } const stdout = (charCode) => { - if (charCode) { - postMessage({ - type: 'stdout', - stdout: charCode, - }) - } else { - console.log(typeof charCode, charCode) - } -} + if (charCode) { + postMessage({ + type: "stdout", + stdout: charCode, + }); + } else { + console.log(typeof charCode, charCode); + } +}; const stderr = (charCode) => { - if (charCode) { - postMessage({ - type: 'stderr', - stderr: charCode, - }) - } else { - console.log(typeof charCode, charCode) - } -} + if (charCode) { + postMessage({ + type: "stderr", + stderr: charCode, + }); + } else { + console.log(typeof charCode, charCode); + } +}; -const stdinBuffer = new StdinBuffer() +const stdinBuffer = new StdinBuffer(); const emscriptenSettings = { - noInitialRun: true, - stdin: stdinBuffer.stdin, - stdout: stdout, - stderr: stderr, - onRuntimeInitialized: () => { - postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab}) - }, - async preRun(Module) { - const versionHex = Module.HEAPU32[Module._Py_Version/4].toString(16); - const versionTuple = versionHex.padStart(8, "0").match(/.{1,2}/g).map((x) => parseInt(x, 16)); - const [major, minor, ..._] = versionTuple; - // Prevent complaints about not finding exec-prefix by making a lib-dynload directory - Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); - Module.addRunDependency("install-stdlib"); - const resp = await fetch(`python${major}.${minor}.zip`); - const stdlibBuffer = await resp.arrayBuffer(); - Module.FS.writeFile(`/lib/python${major}${minor}.zip`, new Uint8Array(stdlibBuffer), { canOwn: true }); - Module.removeRunDependency("install-stdlib"); - } -} + noInitialRun: true, + stdin: stdinBuffer.stdin, + stdout: stdout, + stderr: stderr, + onRuntimeInitialized: () => { + postMessage({ type: "ready", stdinBuffer: stdinBuffer.sab }); + }, + async preRun(Module) { + const versionHex = Module.HEAPU32[Module._Py_Version / 4].toString(16); + const versionTuple = versionHex + .padStart(8, "0") + .match(/.{1,2}/g) + .map((x) => parseInt(x, 16)); + const [major, minor, ..._] = versionTuple; + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); + Module.addRunDependency("install-stdlib"); + const resp = await fetch(`python${major}.${minor}.zip`); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile( + `/lib/python${major}${minor}.zip`, + new Uint8Array(stdlibBuffer), + { canOwn: true }, + ); + Module.removeRunDependency("install-stdlib"); + }, +}; const modulePromise = createEmscriptenModule(emscriptenSettings); - onmessage = async (event) => { - if (event.data.type === 'run') { - const Module = await modulePromise; - if (event.data.files) { - for (const [filename, contents] of Object.entries(event.data.files)) { - Module.FS.writeFile(filename, contents) - } - } - const ret = Module.callMain(event.data.args); - postMessage({ - type: 'finished', - returnCode: ret - }) + if (event.data.type === "run") { + const Module = await modulePromise; + if (event.data.files) { + for (const [filename, contents] of Object.entries(event.data.files)) { + Module.FS.writeFile(filename, contents); + } } -} - + const ret = Module.callMain(event.data.args); + postMessage({ + type: "finished", + returnCode: ret, + }); + } +}; From 43634fc1fcc88b35171aa79258f767ba6477f764 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 5 Dec 2024 01:26:25 +0100 Subject: [PATCH 105/427] gh-127146: Emscripten: Skip segfaults in test suite (#127151) Added skips for tests known to cause problems when running on Emscripten. These mostly relate to the limited stack depth on Emscripten. --- Lib/test/list_tests.py | 3 ++- Lib/test/mapping_tests.py | 3 ++- Lib/test/support/__init__.py | 3 +++ Lib/test/test_ast/test_ast.py | 5 ++++- Lib/test/test_call.py | 3 ++- Lib/test/test_capi/test_misc.py | 1 + Lib/test/test_class.py | 3 ++- Lib/test/test_compile.py | 2 ++ Lib/test/test_copy.py | 3 +++ Lib/test/test_descr.py | 3 +++ Lib/test/test_dict.py | 1 + Lib/test/test_dictviews.py | 3 ++- Lib/test/test_exception_group.py | 4 +++- Lib/test/test_functools.py | 2 ++ Lib/test/test_isinstance.py | 3 +++ Lib/test/test_json/test_recursion.py | 3 +++ Lib/test/test_pathlib/test_pathlib_abc.py | 4 +++- Lib/test/test_traceback.py | 2 ++ Lib/test/test_xml_etree_c.py | 1 + configure | 1 + configure.ac | 1 + 21 files changed, 46 insertions(+), 8 deletions(-) diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index dbc5ef4f9f2cd5..dbd9f27872962d 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit +from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit, skip_emscripten_stack_overflow class CommonTest(seq_tests.CommonTest): @@ -59,6 +59,7 @@ def test_repr(self): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + @skip_emscripten_stack_overflow() def test_repr_deep(self): a = self.type2test([]) for i in range(get_c_recursion_limit() + 1): diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index ed89a81a6ea685..f249f0021e9c1c 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -1,7 +1,7 @@ # tests common to dict and UserDict import unittest import collections -from test.support import get_c_recursion_limit +from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow class BasicTestMappingProtocol(unittest.TestCase): @@ -622,6 +622,7 @@ def __repr__(self): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) + @skip_emscripten_stack_overflow() def test_repr_deep(self): d = self._empty_mapping() for i in range(get_c_recursion_limit() + 1): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2ad267e3e08f0f..5c738ffaa27713 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -535,6 +535,9 @@ def skip_android_selinux(name): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" +def skip_emscripten_stack_overflow(): + return unittest.skipIf(is_emscripten, "Exhausts limited stack on Emscripten") + is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} is_apple = is_apple_mobile or sys.platform == "darwin" diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 67ab8cf6baf657..c268a1f00f938e 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -18,7 +18,7 @@ _testinternalcapi = None from test import support -from test.support import os_helper, script_helper +from test.support import os_helper, script_helper, skip_emscripten_stack_overflow from test.support.ast_helper import ASTTestMixin from test.test_ast.utils import to_tuple from test.test_ast.snippets import ( @@ -745,6 +745,7 @@ def next(self): enum._test_simple_enum(_Precedence, ast._Precedence) @support.cpython_only + @skip_emscripten_stack_overflow() def test_ast_recursion_limit(self): fail_depth = support.exceeds_recursion_limit() crash_depth = 100_000 @@ -1661,6 +1662,7 @@ def test_level_as_none(self): exec(code, ns) self.assertIn('sleep', ns) + @skip_emscripten_stack_overflow() def test_recursion_direct(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e @@ -1668,6 +1670,7 @@ def test_recursion_direct(self): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") + @skip_emscripten_stack_overflow() def test_recursion_indirect(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 9d5256b566b8af..78a706436aea0e 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,6 +1,6 @@ import unittest from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG, - set_recursion_limit, skip_on_s390x) + set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow) try: import _testcapi except ImportError: @@ -1038,6 +1038,7 @@ class TestRecursion(unittest.TestCase): @skip_on_s390x @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") @unittest.skipIf(_testcapi is None, "requires _testcapi") + @skip_emscripten_stack_overflow() def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 80e705a37c4c5e..8e0271919cc8a5 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2137,6 +2137,7 @@ def test_py_config_isoloated_per_interpreter(self): # test fails, assume that the environment in this process may # be altered and suspect. + @requires_subinterpreters @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_configured_settings(self): """ diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 7720cf157fa9ae..e20e59944e9ce9 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,7 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest -from test.support import cpython_only, import_helper, script_helper +from test.support import cpython_only, import_helper, script_helper, skip_emscripten_stack_overflow testmeths = [ @@ -554,6 +554,7 @@ class Custom: self.assertFalse(hasattr(o, "__call__")) self.assertFalse(hasattr(c, "__call__")) + @skip_emscripten_stack_overflow() def testSFBug532646(self): # Test for SF bug 532646 diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index f7ea923ef17672..b5cf2ad18fe60b 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -121,6 +121,7 @@ def __getitem__(self, key): self.assertEqual(d['z'], 12) @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_extended_arg(self): repeat = int(get_c_recursion_limit() * 0.9) longexpr = 'x = x or ' + '-x' * repeat @@ -709,6 +710,7 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_compiler_recursion_limit(self): # Expected limit is Py_C_RECURSION_LIMIT limit = get_c_recursion_limit() diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 3dec64cc9a2414..d76341417e9bef 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -371,6 +371,7 @@ def test_deepcopy_list(self): self.assertIsNot(x, y) self.assertIsNot(x[0], y[0]) + @support.skip_emscripten_stack_overflow() def test_deepcopy_reflexive_list(self): x = [] x.append(x) @@ -398,6 +399,7 @@ def test_deepcopy_tuple_of_immutables(self): y = copy.deepcopy(x) self.assertIs(x, y) + @support.skip_emscripten_stack_overflow() def test_deepcopy_reflexive_tuple(self): x = ([],) x[0].append(x) @@ -415,6 +417,7 @@ def test_deepcopy_dict(self): self.assertIsNot(x, y) self.assertIsNot(x["foo"], y["foo"]) + @support.skip_emscripten_stack_overflow() def test_deepcopy_reflexive_dict(self): x = {} x['foo'] = x diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index aa801b9c4f7ad9..168b78a477ee9c 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -3663,6 +3663,7 @@ def f(a): return a encoding='latin1', errors='replace') self.assertEqual(ba, b'abc\xbd?') + @support.skip_emscripten_stack_overflow() def test_recursive_call(self): # Testing recursive __call__() by setting to instance of class... class A(object): @@ -3942,6 +3943,7 @@ def __del__(self): # it as a leak. del C.__del__ + @unittest.skipIf(support.is_emscripten, "Seems to works in Pyodide?") def test_slots_trash(self): # Testing slot trash... # Deallocating deeply nested slotted trash caused stack overflows @@ -4864,6 +4866,7 @@ class Thing: # CALL_METHOD_DESCRIPTOR_O deque.append(thing, thing) + @support.skip_emscripten_stack_overflow() def test_repr_as_str(self): # Issue #11603: crash or infinite loop when rebinding __str__ as # __repr__. diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index c94dc2df4f0a7f..86b2f22dee5347 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -594,6 +594,7 @@ def __repr__(self): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) + @support.skip_emscripten_stack_overflow() def test_repr_deep(self): d = {} for i in range(get_c_recursion_limit() + 1): diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index d9881611c19c43..d6bf00eeeb0013 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -2,7 +2,7 @@ import copy import pickle import unittest -from test.support import get_c_recursion_limit +from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow class DictSetTest(unittest.TestCase): @@ -277,6 +277,7 @@ def test_recursive_repr(self): # Again. self.assertIsInstance(r, str) + @skip_emscripten_stack_overflow() def test_deeply_nested_repr(self): d = {} for i in range(get_c_recursion_limit()//2 + 100): diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index b4fc290b1f32b6..53212529c27e28 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest -from test.support import get_c_recursion_limit +from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -464,11 +464,13 @@ def make_deep_eg(self): e = ExceptionGroup('eg', [e]) return e + @skip_emscripten_stack_overflow() def test_deep_split(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): e.split(TypeError) + @skip_emscripten_stack_overflow() def test_deep_subgroup(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 6d60f6941c4c5d..ffd2adb8665b45 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -404,6 +404,7 @@ def test_setstate_subclasses(self): self.assertEqual(r, ((1, 2), {})) self.assertIs(type(r[0]), tuple) + @support.skip_emscripten_stack_overflow() def test_recursive_pickle(self): with replaced_module('functools', self.module): f = self.partial(capture) @@ -2054,6 +2055,7 @@ def orig(a, /, b, c=True): ... @support.skip_on_s390x @unittest.skipIf(support.is_wasi, "WASI has limited C stack") + @support.skip_emscripten_stack_overflow() def test_lru_recursion(self): @self.module.lru_cache diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 95a119ba683e09..abc75c82375d98 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -263,12 +263,14 @@ def test_subclass_tuple(self): self.assertEqual(True, issubclass(int, (int, (float, int)))) self.assertEqual(True, issubclass(str, (str, (Child, str)))) + @support.skip_emscripten_stack_overflow() def test_subclass_recursion_limit(self): # make sure that issubclass raises RecursionError before the C stack is # blown with support.infinite_recursion(): self.assertRaises(RecursionError, blowstack, issubclass, str, str) + @support.skip_emscripten_stack_overflow() def test_isinstance_recursion_limit(self): # make sure that issubclass raises RecursionError before the C stack is # blown @@ -315,6 +317,7 @@ def __bases__(self): self.assertRaises(RecursionError, issubclass, int, X()) self.assertRaises(RecursionError, isinstance, 1, X()) + @support.skip_emscripten_stack_overflow() def test_infinite_recursion_via_bases_tuple(self): """Regression test for bpo-30570.""" class Failure(object): diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 290207e9c15b88..663c0643579ac8 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -68,6 +68,7 @@ def default(self, o): self.fail("didn't raise ValueError on default recursion") + @support.skip_emscripten_stack_overflow() def test_highly_nested_objects_decoding(self): # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 @@ -81,6 +82,7 @@ def test_highly_nested_objects_decoding(self): with support.infinite_recursion(): self.loads('[' * 100000 + '1' + ']' * 100000) + @support.skip_emscripten_stack_overflow() def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} @@ -93,6 +95,7 @@ def test_highly_nested_objects_encoding(self): with support.infinite_recursion(5000): self.dumps(d) + @support.skip_emscripten_stack_overflow() def test_endless_recursion(self): # See #12051 class EndlessJSONEncoder(self.json.JSONEncoder): diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index af94ac039808f0..5fa2f550cefcf4 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -9,7 +9,7 @@ from pathlib._abc import UnsupportedOperation, ParserBase, PurePathBase, PathBase import posixpath -from test.support import is_wasi +from test.support import is_wasi, is_emscripten from test.support.os_helper import TESTFN @@ -2298,6 +2298,7 @@ def _check(path, pattern, case_sensitive, expected): _check(path, "dirb/file*", False, ["dirB/fileB"]) @needs_symlinks + @unittest.skipIf(is_emscripten, "Hangs") def test_glob_recurse_symlinks_common(self): def _check(path, glob, expected): actual = {path for path in path.glob(glob, recurse_symlinks=True) @@ -2393,6 +2394,7 @@ def test_rglob_windows(self): self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) @needs_symlinks + @unittest.skipIf(is_emscripten, "Hangs") def test_rglob_recurse_symlinks_common(self): def _check(path, glob, expected): actual = {path for path in path.rglob(glob, recurse_symlinks=True) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index ea8d9f2137aca5..31f0a61d6a9d59 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -2097,6 +2097,7 @@ def deep_eg(self): return e @cpython_only + @support.skip_emscripten_stack_overflow() def test_exception_group_deep_recursion_capi(self): from _testcapi import exception_print LIMIT = 75 @@ -2108,6 +2109,7 @@ def test_exception_group_deep_recursion_capi(self): self.assertIn('ExceptionGroup', output) self.assertLessEqual(output.count('ExceptionGroup'), LIMIT) + @support.skip_emscripten_stack_overflow() def test_exception_group_deep_recursion_traceback(self): LIMIT = 75 eg = self.deep_eg() diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index 3a0fc572f457ff..db19af419bdeab 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -57,6 +57,7 @@ def test_del_attribute(self): del element.attrib self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) + @unittest.skipIf(support.is_emscripten, "segfaults") def test_trashcan(self): # If this test fails, it will most likely die via segfault. e = root = cET.Element('root') diff --git a/configure b/configure index 7efda041ae69d4..c6790777793566 100755 --- a/configure +++ b/configure @@ -9436,6 +9436,7 @@ fi as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,callMain" as_fn_append LDFLAGS_NODIST " -sEXPORTED_FUNCTIONS=_main,_Py_Version" + as_fn_append LDFLAGS_NODIST " -sSTACK_SIZE=5MB" if test "x$enable_wasm_dynamic_linking" = xyes then : diff --git a/configure.ac b/configure.ac index 15f7d07f22473b..9648e438cc7424 100644 --- a/configure.ac +++ b/configure.ac @@ -2334,6 +2334,7 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sSTACK_SIZE=5MB"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) From 87faf0a9c4aa7f8eb5b6b6c8f6e8f5f99b1e3d9b Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 5 Dec 2024 01:44:50 +0100 Subject: [PATCH 106/427] gh-127503: Emscripten make Python.sh function as proper Python CLI (#127506) Modifies the python.sh script to work on macOS, and adapt to recent emscripten changes. --- Tools/wasm/emscripten/__main__.py | 21 ++++++++++++-- Tools/wasm/emscripten/node_entry.mjs | 43 +++++++++++++++++++--------- configure | 2 +- configure.ac | 2 +- 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 9ce8dd6a364ad6..c998ed71309dad 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -218,9 +218,26 @@ def configure_emscripten_python(context, working_dir): f"""\ #!/bin/sh + # Macs come with FreeBSD coreutils which doesn't have the -s option + # so feature detect and work around it. + if which grealpath > /dev/null; then + # It has brew installed gnu core utils, use that + REALPATH="grealpath -s" + elif which realpath > /dev/null && realpath --version 2&>1 | grep GNU > /dev/null; then + # realpath points to GNU realpath so use it. + REALPATH="realpath -s" + else + # Shim for macs without GNU coreutils + abs_path () {{ + echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" + }} + REALPATH=abs_path + fi + # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. - exec {host_runner} {node_entry} "$(realpath -s $0)" "$@" + # Intentionally allow word splitting on NODEFLAGS. + exec {host_runner} $NODEFLAGS {node_entry} --this-program="$($REALPATH "$0")" "$@" """ ) ) @@ -233,7 +250,7 @@ def configure_emscripten_python(context, working_dir): def make_emscripten_python(context, working_dir): """Run `make` for the emscripten/host build.""" call( - ["make", "--jobs", str(cpu_count()), "commoninstall"], + ["make", "--jobs", str(cpu_count()), "all"], env=updated_env(), quiet=context.quiet, ) diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs index cb1c6ff3cba6aa..40ab1515cf28c1 100644 --- a/Tools/wasm/emscripten/node_entry.mjs +++ b/Tools/wasm/emscripten/node_entry.mjs @@ -1,30 +1,47 @@ import EmscriptenModule from "./python.mjs"; -import { dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import fs from "node:fs"; if (process?.versions?.node) { const nodeVersion = Number(process.versions.node.split(".", 1)[0]); if (nodeVersion < 18) { - process.stderr.write( - `Node version must be >= 18, got version ${process.version}\n`, - ); - process.exit(1); + process.stderr.write( + `Node version must be >= 18, got version ${process.version}\n`, + ); + process.exit(1); } } +function rootDirsToMount(Module) { + return fs + .readdirSync("/") + .filter((dir) => !["dev", "lib", "proc"].includes(dir)) + .map((dir) => "/" + dir); +} + +function mountDirectories(Module) { + for (const dir of rootDirsToMount(Module)) { + Module.FS.mkdirTree(dir); + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir); + } +} + +const thisProgram = "--this-program="; +const thisProgramIndex = process.argv.findIndex((x) => + x.startsWith(thisProgram), +); + const settings = { preRun(Module) { - const __dirname = dirname(fileURLToPath(import.meta.url)); - Module.FS.mkdirTree("/lib/"); - Module.FS.mount(Module.FS.filesystems.NODEFS, { root: __dirname + "/lib/" }, "/lib/"); + mountDirectories(Module); + Module.FS.chdir(process.cwd()); + Object.assign(Module.ENV, process.env); }, - // The first three arguments are: "node", path to this file, path to - // python.sh. After that come the arguments the user passed to python.sh. - arguments: process.argv.slice(3), // Ensure that sys.executable, sys._base_executable, etc point to python.sh // not to this file. To properly handle symlinks, python.sh needs to compute // its own path. - thisProgram: process.argv[2], + thisProgram: process.argv[thisProgramIndex], + // After python.sh come the arguments thatthe user passed to python.sh. + arguments: process.argv.slice(thisProgramIndex + 1), }; await EmscriptenModule(settings); diff --git a/configure b/configure index c6790777793566..2fa473b9fe32c0 100755 --- a/configure +++ b/configure @@ -9434,7 +9434,7 @@ fi as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,callMain" + as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV" as_fn_append LDFLAGS_NODIST " -sEXPORTED_FUNCTIONS=_main,_Py_Version" as_fn_append LDFLAGS_NODIST " -sSTACK_SIZE=5MB" diff --git a/configure.ac b/configure.ac index 9648e438cc7424..8ca8e0f7802742 100644 --- a/configure.ac +++ b/configure.ac @@ -2332,7 +2332,7 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -sSTACK_SIZE=5MB"]) From 6cf77949fba7b44f6885794b2028f091f42f5d6c Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Wed, 4 Dec 2024 19:00:20 -0800 Subject: [PATCH 107/427] gh-127434: Fix iOS `xcrun --sdk` clang/ar scripts to allow arguments with spaces (#127575) Added shell escaping to ensure iOS compiler shims can accept arguments with spaces. --- .../2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst | 1 + iOS/Resources/bin/arm64-apple-ios-ar | 2 +- iOS/Resources/bin/arm64-apple-ios-clang | 2 +- iOS/Resources/bin/arm64-apple-ios-clang++ | 2 +- iOS/Resources/bin/arm64-apple-ios-cpp | 2 +- iOS/Resources/bin/arm64-apple-ios-simulator-ar | 2 +- iOS/Resources/bin/arm64-apple-ios-simulator-clang | 2 +- iOS/Resources/bin/arm64-apple-ios-simulator-clang++ | 2 +- iOS/Resources/bin/arm64-apple-ios-simulator-cpp | 2 +- iOS/Resources/bin/x86_64-apple-ios-simulator-ar | 2 +- iOS/Resources/bin/x86_64-apple-ios-simulator-clang | 2 +- iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ | 2 +- iOS/Resources/bin/x86_64-apple-ios-simulator-cpp | 2 +- 13 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst new file mode 100644 index 00000000000000..08b27a7890bb1c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst @@ -0,0 +1 @@ +The iOS compiler shims can now accept arguments with spaces. diff --git a/iOS/Resources/bin/arm64-apple-ios-ar b/iOS/Resources/bin/arm64-apple-ios-ar index 8122332b9c1de0..3cf3eb218741fa 100755 --- a/iOS/Resources/bin/arm64-apple-ios-ar +++ b/iOS/Resources/bin/arm64-apple-ios-ar @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} ar $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-clang b/iOS/Resources/bin/arm64-apple-ios-clang index 4d525751eba798..c39519cd1f8c94 100755 --- a/iOS/Resources/bin/arm64-apple-ios-clang +++ b/iOS/Resources/bin/arm64-apple-ios-clang @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-clang++ b/iOS/Resources/bin/arm64-apple-ios-clang++ index f24bec11268f7e..d9b12925f384b9 100755 --- a/iOS/Resources/bin/arm64-apple-ios-clang++ +++ b/iOS/Resources/bin/arm64-apple-ios-clang++ @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-cpp b/iOS/Resources/bin/arm64-apple-ios-cpp index 891bb25bb4318c..24da23d3448ae0 100755 --- a/iOS/Resources/bin/arm64-apple-ios-cpp +++ b/iOS/Resources/bin/arm64-apple-ios-cpp @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/iOS/Resources/bin/arm64-apple-ios-simulator-ar index 74ed3bc6df1c2b..b836b6db9025bb 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-ar +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/iOS/Resources/bin/arm64-apple-ios-simulator-clang index 32574cad284441..92e8d853d6ebc3 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-clang +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ index ef37d05b512959..076469cc70cf98 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios-simulator $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp index 6aaf6fbe188c32..c57f28cee5bcfe 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar index 74ed3bc6df1c2b..b836b6db9025bb 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang index bcbe91f6061e16..17cbe0c8a1e213 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ index 86f03ea32bc2fd..565d47b24c214b 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios-simulator $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp index e6a42d9b85dec7..63fc8e8de2d38d 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E "$@" From 1ef6e8ca3faf2c2b008fb170c7c44c38b86e874a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Dec 2024 10:37:14 +0100 Subject: [PATCH 108/427] gh-119182: Complete PyUnicodeWriter documentation (#127607) --- Doc/c-api/unicode.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 59bd7661965d93..dcbc8804cd6b89 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1588,6 +1588,11 @@ object. Create a Unicode writer instance. + *length* must be greater than or equal to ``0``. + + If *length* is greater than ``0``, preallocate an internal buffer of + *length* characters. + Set an exception and return ``NULL`` on error. .. c:function:: PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) @@ -1596,12 +1601,16 @@ object. Set an exception and return ``NULL`` on error. + The writer instance is invalid after this call. + .. c:function:: void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) Discard the internal Unicode buffer and destroy the writer instance. If *writer* is ``NULL``, no operation is performed. + The writer instance is invalid after this call. + .. c:function:: int PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) Write the single Unicode character *ch* into *writer*. From fcbe6ecdb6ed4dd93b2ee144f89a73af755e2634 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Dec 2024 10:39:44 +0100 Subject: [PATCH 109/427] gh-93312: Include to get PIDFD_NONBLOCK (#127593) --- .../Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst | 2 ++ Modules/posixmodule.c | 3 +++ configure | 6 ++++++ configure.ac | 2 +- pyconfig.h.in | 3 +++ 5 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst diff --git a/Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst b/Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst new file mode 100644 index 00000000000000..e245fa2bdd00b4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst @@ -0,0 +1,2 @@ +Include ```` to get ``os.PIDFD_NONBLOCK`` constant. Patch by +Victor Stinner. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6eb7054b566e3f..2c26fbeac9a1be 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -73,6 +73,9 @@ #ifdef HAVE_SYS_TIME_H # include // futimes() #endif +#ifdef HAVE_SYS_PIDFD_H +# include // PIDFD_NONBLOCK +#endif // SGI apparently needs this forward declaration diff --git a/configure b/configure index 2fa473b9fe32c0..5e9bcb602d884e 100755 --- a/configure +++ b/configure @@ -11178,6 +11178,12 @@ if test "x$ac_cv_header_sys_param_h" = xyes then : printf "%s\n" "#define HAVE_SYS_PARAM_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "sys/pidfd.h" "ac_cv_header_sys_pidfd_h" "$ac_includes_default" +if test "x$ac_cv_header_sys_pidfd_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_PIDFD_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "sys/poll.h" "ac_cv_header_sys_poll_h" "$ac_includes_default" if test "x$ac_cv_header_sys_poll_h" = xyes diff --git a/configure.ac b/configure.ac index 8ca8e0f7802742..bf3685e1b1b209 100644 --- a/configure.ac +++ b/configure.ac @@ -2932,7 +2932,7 @@ AC_CHECK_HEADERS([ \ linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ - sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/poll.h \ + sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/pidfd.h sys/poll.h \ sys/random.h sys/resource.h sys/select.h sys/sendfile.h sys/socket.h sys/soundcard.h sys/stat.h \ sys/statvfs.h sys/sys_domain.h sys/syscall.h sys/sysmacros.h sys/termio.h sys/time.h sys/times.h sys/timerfd.h \ sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h sys/xattr.h sysexits.h syslog.h \ diff --git a/pyconfig.h.in b/pyconfig.h.in index 924d86627b0e9b..6a1f1284650b9f 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1388,6 +1388,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_PARAM_H +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_PIDFD_H + /* Define to 1 if you have the header file. */ #undef HAVE_SYS_POLL_H From 67b9a5331ae45aa126877d7f96a1e235600f9c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:01:59 +0100 Subject: [PATCH 110/427] gh-127413: allow to show specialized bytecode via `dis` CLI (#127414) --- Doc/library/dis.rst | 8 +++++++- Doc/whatsnew/3.14.rst | 6 ++++++ Lib/dis.py | 5 ++++- .../2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst | 2 ++ 4 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index e2926f2440af6d..f8f4188d27b472 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -60,6 +60,8 @@ interpreter. The :option:`-P ` command-line option and the ``show_positions`` argument were added. + The :option:`-S ` command-line option is added. + Example: Given the function :func:`!myfunc`:: def myfunc(alist): @@ -89,7 +91,7 @@ The :mod:`dis` module can be invoked as a script from the command line: .. code-block:: sh - python -m dis [-h] [-C] [-O] [-P] [infile] + python -m dis [-h] [-C] [-O] [-P] [-S] [infile] The following options are accepted: @@ -111,6 +113,10 @@ The following options are accepted: Show positions of instructions in the source code. +.. cmdoption:: -S, --specialized + + Show specialized bytecode. + If :file:`infile` is specified, its disassembled code will be written to stdout. Otherwise, disassembly is performed on compiled source code received from stdin. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 52a6d6e4340194..e83c509a025ab5 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -348,12 +348,18 @@ dis This feature is also exposed via :option:`dis --show-positions`. (Contributed by Bénédikt Tran in :gh:`123165`.) +* Add the :option:`dis --specialized` command-line option to + show specialized bytecode. + (Contributed by Bénédikt Tran in :gh:`127413`.) + + errno ----- * Add :data:`errno.EHWPOISON` error code. (Contributed by James Roy in :gh:`126585`.) + fractions --------- diff --git a/Lib/dis.py b/Lib/dis.py index 1718e39cceb638..6b3e9ef8399e1c 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -1125,6 +1125,8 @@ def main(): help='show instruction offsets') parser.add_argument('-P', '--show-positions', action='store_true', help='show instruction positions') + parser.add_argument('-S', '--specialized', action='store_true', + help='show specialized bytecode') parser.add_argument('infile', nargs='?', default='-') args = parser.parse_args() if args.infile == '-': @@ -1135,7 +1137,8 @@ def main(): with open(args.infile, 'rb') as infile: source = infile.read() code = compile(source, name, "exec") - dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets, show_positions=args.show_positions) + dis(code, show_caches=args.show_caches, adaptive=args.specialized, + show_offsets=args.show_offsets, show_positions=args.show_positions) if __name__ == "__main__": main() diff --git a/Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst b/Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst new file mode 100644 index 00000000000000..2330fb66253265 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst @@ -0,0 +1,2 @@ +Add the :option:`dis --specialized` command-line option to show specialized +bytecode. Patch by Bénédikt Tran. From 208b0fb645c0e14b0826c0014e74a0b70c58c9d6 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 5 Dec 2024 11:07:38 -0500 Subject: [PATCH 111/427] gh-122431: Disallow negative values in `readline.append_history_file` (#122469) Co-authored-by: Victor Stinner --- Lib/test/test_readline.py | 8 ++++++++ .../2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst | 1 + Modules/readline.c | 6 ++++++ 3 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 50e77cbbb6be13..8b8772c66ee654 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -114,6 +114,14 @@ def test_write_read_append(self): # write_history_file can create the target readline.write_history_file(hfilename) + # Negative values should be disallowed + with self.assertRaises(ValueError): + readline.append_history_file(-42, hfilename) + + # See gh-122431, using the minimum signed integer value caused a segfault + with self.assertRaises(ValueError): + readline.append_history_file(-2147483648, hfilename) + def test_nonascii_history(self): readline.clear_history() try: diff --git a/Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst b/Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst new file mode 100644 index 00000000000000..16ad75792aefa2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst @@ -0,0 +1 @@ +:func:`readline.append_history_file` now raises a :exc:`ValueError` when given a negative value. diff --git a/Modules/readline.c b/Modules/readline.c index 35655c70a4618f..7d1f703f7dbdde 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -351,6 +351,12 @@ readline_append_history_file_impl(PyObject *module, int nelements, PyObject *filename_obj) /*[clinic end generated code: output=5df06fc9da56e4e4 input=784b774db3a4b7c5]*/ { + if (nelements < 0) + { + PyErr_SetString(PyExc_ValueError, "nelements must be positive"); + return NULL; + } + PyObject *filename_bytes; const char *filename; int err; From d958d9f4a1b71c6d30960bf6c53c41046ea94590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Thu, 5 Dec 2024 19:43:19 +0100 Subject: [PATCH 112/427] GH-126727: Fix test_era_nl_langinfo with Japanese ERAs on Solaris (GH-127327) Fix test_era_nl_langinfo with Japanese ERAs on Solaris --- Lib/test/test__locale.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 2c751033ebb3e2..cef84fd9580c37 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -102,6 +102,11 @@ def accept(loc): # ps_AF doesn't work on Windows: see bpo-38324 (msg361830) del known_numerics['ps_AF'] +if sys.platform == 'sunos5': + # On Solaris, Japanese ERAs start with the year 1927, + # and thus there's less of them. + known_era['ja_JP'] = (5, '+:1:2019/05/01:2019/12/31:令和:%EC元年') + class _LocaleTests(unittest.TestCase): def setUp(self): From 23f2e8f13c4e4a34106cf96fad9329cbfbf8844d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 5 Dec 2024 21:10:46 +0200 Subject: [PATCH 113/427] gh-127221: Add colour to unittest output (#127223) Co-authored-by: Kirill Podoprigora --- Doc/conf.py | 7 ++ Doc/library/doctest.rst | 4 + Doc/library/traceback.rst | 4 + Doc/library/unittest.rst | 4 +- Doc/using/cmdline.rst | 8 -- Doc/whatsnew/3.13.rst | 9 -- Doc/whatsnew/3.14.rst | 7 ++ Lib/test/test_unittest/test_async_case.py | 2 + Lib/test/test_unittest/test_program.py | 6 + Lib/test/test_unittest/test_result.py | 16 ++- Lib/test/test_unittest/test_runner.py | 13 +++ Lib/test/test_unittest/test_skipping.py | 3 + Lib/unittest/result.py | 4 +- Lib/unittest/runner.py | 108 +++++++++++------- ...-11-23-00-17-29.gh-issue-127221.OSXdFE.rst | 1 + 15 files changed, 136 insertions(+), 60 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst diff --git a/Doc/conf.py b/Doc/conf.py index 738c9901eef06f..9cde394cbaed69 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -78,6 +78,13 @@ .. |python_version_literal| replace:: ``Python {version}`` .. |python_x_dot_y_literal| replace:: ``python{version}`` .. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}`` + +.. Apparently this how you hack together a formatted link: + (https://www.docutils.org/docs/ref/rst/directives.html#replacement-text) +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ """ # There are two options for replacing |today|. Either, you set today to some diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 6b0282eed49566..106b0a6c95b7be 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -136,6 +136,10 @@ examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file :file:`Lib/test/test_doctest/test_doctest.py`. +.. versionadded:: 3.13 + Output is colorized by default and can be + :ref:`controlled using environment variables `. + .. _doctest-simple-testmod: diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 8f94fc448f2482..4899ed64ebad8d 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -44,6 +44,10 @@ The module's API can be divided into two parts: necessary for later formatting without holding references to actual exception and traceback objects. +.. versionadded:: 3.13 + Output is colorized by default and can be + :ref:`controlled using environment variables `. + Module-Level Functions ---------------------- diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 38bad9405597dd..7f8b710f611002 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -46,7 +46,6 @@ test runner a textual interface, or return a special value to indicate the results of executing the tests. - .. seealso:: Module :mod:`doctest` @@ -198,6 +197,9 @@ For a list of all the command-line options:: In earlier versions it was only possible to run individual test methods and not modules or classes. +.. versionadded:: 3.14 + Output is colorized by default and can be + :ref:`controlled using environment variables `. Command-line options ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 6cf42b27718022..7db2f4820f346a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -663,14 +663,6 @@ output. To control the color output only in the Python interpreter, the precedence over ``NO_COLOR``, which in turn takes precedence over ``FORCE_COLOR``. -.. Apparently this how you hack together a formatted link: - -.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` -.. _FORCE_COLOR: https://force-color.org/ - -.. |NO_COLOR| replace:: ``NO_COLOR`` -.. _NO_COLOR: https://no-color.org/ - Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 664b1866172378..9f6d98b9950d19 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -252,15 +252,6 @@ Improved error messages the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment variables. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) -.. Apparently this how you hack together a formatted link: - (https://www.docutils.org/docs/ref/rst/directives.html#replacement-text) - -.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` -.. _FORCE_COLOR: https://force-color.org/ - -.. |NO_COLOR| replace:: ``NO_COLOR`` -.. _NO_COLOR: https://no-color.org/ - * A common mistake is to write a script with the same name as a standard library module. When this results in errors, we now display a more helpful error message: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index e83c509a025ab5..db25c037e509b6 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -616,6 +616,13 @@ unicodedata unittest -------- +* :mod:`unittest` output is now colored by default. + This can be controlled via the :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ + and |FORCE_COLOR|_ environment variables. + See also :ref:`using-on-controlling-color`. + (Contributed by Hugo van Kemenade in :gh:`127221`.) + * unittest discovery supports :term:`namespace package` as start directory again. It was removed in Python 3.11. (Contributed by Jacob Walls in :gh:`80958`.) diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 00ef55bdf9bc83..8ea244bff05c5f 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -2,6 +2,7 @@ import contextvars import unittest from test import support +from test.support import force_not_colorized support.requires_working_socket(module=True) @@ -252,6 +253,7 @@ async def on_cleanup(self): test.doCleanups() self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup']) + @force_not_colorized def test_exception_in_tear_clean_up(self): class Test(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 7241cf59f73d4f..0b46f338ac77e1 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -4,6 +4,7 @@ from test import support import unittest import test.test_unittest +from test.support import force_not_colorized from test.test_unittest.test_result import BufferedWriter @@ -120,6 +121,7 @@ def run(self, test): self.assertEqual(['test.test_unittest', 'test.test_unittest2'], program.testNames) + @force_not_colorized def test_NonExit(self): stream = BufferedWriter() program = unittest.main(exit=False, @@ -135,6 +137,7 @@ def test_NonExit(self): 'expected failures=1, unexpected successes=1)\n') self.assertTrue(out.endswith(expected)) + @force_not_colorized def test_Exit(self): stream = BufferedWriter() with self.assertRaises(SystemExit) as cm: @@ -152,6 +155,7 @@ def test_Exit(self): 'expected failures=1, unexpected successes=1)\n') self.assertTrue(out.endswith(expected)) + @force_not_colorized def test_ExitAsDefault(self): stream = BufferedWriter() with self.assertRaises(SystemExit): @@ -167,6 +171,7 @@ def test_ExitAsDefault(self): 'expected failures=1, unexpected successes=1)\n') self.assertTrue(out.endswith(expected)) + @force_not_colorized def test_ExitSkippedSuite(self): stream = BufferedWriter() with self.assertRaises(SystemExit) as cm: @@ -179,6 +184,7 @@ def test_ExitSkippedSuite(self): expected = '\n\nOK (skipped=1)\n' self.assertTrue(out.endswith(expected)) + @force_not_colorized def test_ExitEmptySuite(self): stream = BufferedWriter() with self.assertRaises(SystemExit) as cm: diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 4e5ec54e9c892a..746b9fa2677717 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -7,6 +7,7 @@ import traceback import unittest from unittest.util import strclass +from test.support import force_not_colorized from test.test_unittest.support import BufferedWriter @@ -14,7 +15,7 @@ class MockTraceback(object): class TracebackException: def __init__(self, *args, **kwargs): self.capture_locals = kwargs.get('capture_locals', False) - def format(self): + def format(self, **kwargs): result = ['A traceback'] if self.capture_locals: result.append('locals') @@ -205,6 +206,7 @@ def test_1(self): self.assertIs(test_case, test) self.assertIsInstance(formatted_exc, str) + @force_not_colorized def test_addFailure_filter_traceback_frames(self): class Foo(unittest.TestCase): def test_1(self): @@ -231,6 +233,7 @@ def get_exc_info(): self.assertEqual(len(dropped), 1) self.assertIn("raise self.failureException(msg)", dropped[0]) + @force_not_colorized def test_addFailure_filter_traceback_frames_context(self): class Foo(unittest.TestCase): def test_1(self): @@ -260,6 +263,7 @@ def get_exc_info(): self.assertEqual(len(dropped), 1) self.assertIn("raise self.failureException(msg)", dropped[0]) + @force_not_colorized def test_addFailure_filter_traceback_frames_chained_exception_self_loop(self): class Foo(unittest.TestCase): def test_1(self): @@ -285,6 +289,7 @@ def get_exc_info(): formatted_exc = result.failures[0][1] self.assertEqual(formatted_exc.count("Exception: Loop\n"), 1) + @force_not_colorized def test_addFailure_filter_traceback_frames_chained_exception_cycle(self): class Foo(unittest.TestCase): def test_1(self): @@ -446,6 +451,7 @@ def testFailFast(self): result.addUnexpectedSuccess(None) self.assertTrue(result.shouldStop) + @force_not_colorized def testFailFastSetByRunner(self): stream = BufferedWriter() runner = unittest.TextTestRunner(stream=stream, failfast=True) @@ -619,6 +625,7 @@ def _run_test(self, test_name, verbosity, tearDownError=None): test.run(result) return stream.getvalue() + @force_not_colorized def testDotsOutput(self): self.assertEqual(self._run_test('testSuccess', 1), '.') self.assertEqual(self._run_test('testSkip', 1), 's') @@ -627,6 +634,7 @@ def testDotsOutput(self): self.assertEqual(self._run_test('testExpectedFailure', 1), 'x') self.assertEqual(self._run_test('testUnexpectedSuccess', 1), 'u') + @force_not_colorized def testLongOutput(self): classname = f'{__name__}.{self.Test.__qualname__}' self.assertEqual(self._run_test('testSuccess', 2), @@ -642,17 +650,21 @@ def testLongOutput(self): self.assertEqual(self._run_test('testUnexpectedSuccess', 2), f'testUnexpectedSuccess ({classname}.testUnexpectedSuccess) ... unexpected success\n') + @force_not_colorized def testDotsOutputSubTestSuccess(self): self.assertEqual(self._run_test('testSubTestSuccess', 1), '.') + @force_not_colorized def testLongOutputSubTestSuccess(self): classname = f'{__name__}.{self.Test.__qualname__}' self.assertEqual(self._run_test('testSubTestSuccess', 2), f'testSubTestSuccess ({classname}.testSubTestSuccess) ... ok\n') + @force_not_colorized def testDotsOutputSubTestMixed(self): self.assertEqual(self._run_test('testSubTestMixed', 1), 'sFE') + @force_not_colorized def testLongOutputSubTestMixed(self): classname = f'{__name__}.{self.Test.__qualname__}' self.assertEqual(self._run_test('testSubTestMixed', 2), @@ -661,6 +673,7 @@ def testLongOutputSubTestMixed(self): f' testSubTestMixed ({classname}.testSubTestMixed) [fail] (c=3) ... FAIL\n' f' testSubTestMixed ({classname}.testSubTestMixed) [error] (d=4) ... ERROR\n') + @force_not_colorized def testDotsOutputTearDownFail(self): out = self._run_test('testSuccess', 1, AssertionError('fail')) self.assertEqual(out, 'F') @@ -671,6 +684,7 @@ def testDotsOutputTearDownFail(self): out = self._run_test('testSkip', 1, AssertionError('fail')) self.assertEqual(out, 'sF') + @force_not_colorized def testLongOutputTearDownFail(self): classname = f'{__name__}.{self.Test.__qualname__}' out = self._run_test('testSuccess', 2, AssertionError('fail')) diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index 1b9cef43e3f9c5..1131cd73128866 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -4,6 +4,7 @@ import pickle import subprocess from test import support +from test.support import force_not_colorized import unittest from unittest.case import _Outcome @@ -106,6 +107,7 @@ def cleanup2(*args, **kwargs): self.assertTrue(test.doCleanups()) self.assertEqual(cleanups, [(2, (), {}), (1, (1, 2, 3), dict(four='hello', five='goodbye'))]) + @force_not_colorized def testCleanUpWithErrors(self): class TestableTest(unittest.TestCase): def testNothing(self): @@ -416,6 +418,7 @@ def cleanup2(): self.assertIsInstance(e2[1], CustomError) self.assertEqual(str(e2[1]), 'cleanup1') + @force_not_colorized def test_with_errors_addCleanUp(self): ordering = [] class TestableTest(unittest.TestCase): @@ -439,6 +442,7 @@ def tearDownClass(cls): ['setUpClass', 'setUp', 'cleanup_exc', 'tearDownClass', 'cleanup_good']) + @force_not_colorized def test_run_with_errors_addClassCleanUp(self): ordering = [] class TestableTest(unittest.TestCase): @@ -462,6 +466,7 @@ def tearDownClass(cls): ['setUpClass', 'setUp', 'test', 'cleanup_good', 'tearDownClass', 'cleanup_exc']) + @force_not_colorized def test_with_errors_in_addClassCleanup_and_setUps(self): ordering = [] class_blow_up = False @@ -514,6 +519,7 @@ def tearDownClass(cls): ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) + @force_not_colorized def test_with_errors_in_tearDownClass(self): ordering = [] class TestableTest(unittest.TestCase): @@ -590,6 +596,7 @@ def test(self): 'inner setup', 'inner test', 'inner cleanup', 'end outer test', 'outer cleanup']) + @force_not_colorized def test_run_empty_suite_error_message(self): class EmptyTest(unittest.TestCase): pass @@ -663,6 +670,7 @@ class Module(object): self.assertEqual(cleanups, [((1, 2), {'function': 'hello'})]) + @force_not_colorized def test_run_module_cleanUp(self): blowUp = True ordering = [] @@ -802,6 +810,7 @@ def tearDownClass(cls): 'tearDownClass', 'cleanup_good']) self.assertEqual(unittest.case._module_cleanups, []) + @force_not_colorized def test_run_module_cleanUp_when_teardown_exception(self): ordering = [] class Module(object): @@ -963,6 +972,7 @@ def testNothing(self): self.assertEqual(cleanups, [((1, 2), {'function': 3, 'self': 4})]) + @force_not_colorized def test_with_errors_in_addClassCleanup(self): ordering = [] @@ -996,6 +1006,7 @@ def tearDownClass(cls): ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) + @force_not_colorized def test_with_errors_in_addCleanup(self): ordering = [] class Module(object): @@ -1026,6 +1037,7 @@ def tearDown(self): ['setUpModule', 'setUp', 'test', 'tearDown', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) + @force_not_colorized def test_with_errors_in_addModuleCleanup_and_setUps(self): ordering = [] module_blow_up = False @@ -1318,6 +1330,7 @@ def MockResultClass(*args): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + @force_not_colorized @support.requires_subprocess() def test_warnings(self): """ diff --git a/Lib/test/test_unittest/test_skipping.py b/Lib/test/test_unittest/test_skipping.py index f146dcac18ecc0..f5cb860c60b156 100644 --- a/Lib/test/test_unittest/test_skipping.py +++ b/Lib/test/test_unittest/test_skipping.py @@ -1,5 +1,6 @@ import unittest +from test.support import force_not_colorized from test.test_unittest.support import LoggingResult @@ -293,6 +294,7 @@ def test_die(self): self.assertFalse(result.unexpectedSuccesses) self.assertTrue(result.wasSuccessful()) + @force_not_colorized def test_expected_failure_and_fail_in_cleanup(self): class Foo(unittest.TestCase): @unittest.expectedFailure @@ -372,6 +374,7 @@ def test_die(self): self.assertEqual(result.unexpectedSuccesses, [test]) self.assertFalse(result.wasSuccessful()) + @force_not_colorized def test_unexpected_success_and_fail_in_cleanup(self): class Foo(unittest.TestCase): @unittest.expectedFailure diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 3ace0a5b7bf2ef..97262735aa8311 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -189,7 +189,9 @@ def _exc_info_to_string(self, err, test): tb_e = traceback.TracebackException( exctype, value, tb, capture_locals=self.tb_locals, compact=True) - msgLines = list(tb_e.format()) + from _colorize import can_colorize + + msgLines = list(tb_e.format(colorize=can_colorize())) if self.buffer: output = sys.stdout.getvalue() diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 2bcadf0c998bd9..d60c295a1eddf7 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -4,6 +4,8 @@ import time import warnings +from _colorize import get_colors + from . import result from .case import _SubTest from .signals import registerResult @@ -13,18 +15,18 @@ class _WritelnDecorator(object): """Used to decorate file-like objects with a handy 'writeln' method""" - def __init__(self,stream): + def __init__(self, stream): self.stream = stream def __getattr__(self, attr): if attr in ('stream', '__getstate__'): raise AttributeError(attr) - return getattr(self.stream,attr) + return getattr(self.stream, attr) def writeln(self, arg=None): if arg: self.write(arg) - self.write('\n') # text-mode streams translate to \r\n if needed + self.write('\n') # text-mode streams translate to \r\n if needed class TextTestResult(result.TestResult): @@ -43,6 +45,7 @@ def __init__(self, stream, descriptions, verbosity, *, durations=None): self.showAll = verbosity > 1 self.dots = verbosity == 1 self.descriptions = descriptions + self._ansi = get_colors() self._newline = True self.durations = durations @@ -76,86 +79,102 @@ def _write_status(self, test, status): def addSubTest(self, test, subtest, err): if err is not None: + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: if issubclass(err[0], subtest.failureException): - self._write_status(subtest, "FAIL") + self._write_status(subtest, f"{red}FAIL{reset}") else: - self._write_status(subtest, "ERROR") + self._write_status(subtest, f"{red}ERROR{reset}") elif self.dots: if issubclass(err[0], subtest.failureException): - self.stream.write('F') + self.stream.write(f"{red}F{reset}") else: - self.stream.write('E') + self.stream.write(f"{red}E{reset}") self.stream.flush() super(TextTestResult, self).addSubTest(test, subtest, err) def addSuccess(self, test): super(TextTestResult, self).addSuccess(test) + green, reset = self._ansi.GREEN, self._ansi.RESET if self.showAll: - self._write_status(test, "ok") + self._write_status(test, f"{green}ok{reset}") elif self.dots: - self.stream.write('.') + self.stream.write(f"{green}.{reset}") self.stream.flush() def addError(self, test, err): super(TextTestResult, self).addError(test, err) + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: - self._write_status(test, "ERROR") + self._write_status(test, f"{red}ERROR{reset}") elif self.dots: - self.stream.write('E') + self.stream.write(f"{red}E{reset}") self.stream.flush() def addFailure(self, test, err): super(TextTestResult, self).addFailure(test, err) + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: - self._write_status(test, "FAIL") + self._write_status(test, f"{red}FAIL{reset}") elif self.dots: - self.stream.write('F') + self.stream.write(f"{red}F{reset}") self.stream.flush() def addSkip(self, test, reason): super(TextTestResult, self).addSkip(test, reason) + yellow, reset = self._ansi.YELLOW, self._ansi.RESET if self.showAll: - self._write_status(test, "skipped {0!r}".format(reason)) + self._write_status(test, f"{yellow}skipped{reset} {reason!r}") elif self.dots: - self.stream.write("s") + self.stream.write(f"{yellow}s{reset}") self.stream.flush() def addExpectedFailure(self, test, err): super(TextTestResult, self).addExpectedFailure(test, err) + yellow, reset = self._ansi.YELLOW, self._ansi.RESET if self.showAll: - self.stream.writeln("expected failure") + self.stream.writeln(f"{yellow}expected failure{reset}") self.stream.flush() elif self.dots: - self.stream.write("x") + self.stream.write(f"{yellow}x{reset}") self.stream.flush() def addUnexpectedSuccess(self, test): super(TextTestResult, self).addUnexpectedSuccess(test) + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: - self.stream.writeln("unexpected success") + self.stream.writeln(f"{red}unexpected success{reset}") self.stream.flush() elif self.dots: - self.stream.write("u") + self.stream.write(f"{red}u{reset}") self.stream.flush() def printErrors(self): + bold_red = self._ansi.BOLD_RED + red = self._ansi.RED + reset = self._ansi.RESET if self.dots or self.showAll: self.stream.writeln() self.stream.flush() - self.printErrorList('ERROR', self.errors) - self.printErrorList('FAIL', self.failures) - unexpectedSuccesses = getattr(self, 'unexpectedSuccesses', ()) + self.printErrorList(f"{red}ERROR{reset}", self.errors) + self.printErrorList(f"{red}FAIL{reset}", self.failures) + unexpectedSuccesses = getattr(self, "unexpectedSuccesses", ()) if unexpectedSuccesses: self.stream.writeln(self.separator1) for test in unexpectedSuccesses: - self.stream.writeln(f"UNEXPECTED SUCCESS: {self.getDescription(test)}") + self.stream.writeln( + f"{red}UNEXPECTED SUCCESS{bold_red}: " + f"{self.getDescription(test)}{reset}" + ) self.stream.flush() def printErrorList(self, flavour, errors): + bold_red, reset = self._ansi.BOLD_RED, self._ansi.RESET for test, err in errors: self.stream.writeln(self.separator1) - self.stream.writeln("%s: %s" % (flavour,self.getDescription(test))) + self.stream.writeln( + f"{flavour}{bold_red}: {self.getDescription(test)}{reset}" + ) self.stream.writeln(self.separator2) self.stream.writeln("%s" % err) self.stream.flush() @@ -232,7 +251,7 @@ def run(self, test): if self.warnings: # if self.warnings is set, use it to filter all the warnings warnings.simplefilter(self.warnings) - startTime = time.perf_counter() + start_time = time.perf_counter() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: startTestRun() @@ -242,8 +261,8 @@ def run(self, test): stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() - stopTime = time.perf_counter() - timeTaken = stopTime - startTime + stop_time = time.perf_counter() + time_taken = stop_time - start_time result.printErrors() if self.durations is not None: self._printDurations(result) @@ -253,10 +272,10 @@ def run(self, test): run = result.testsRun self.stream.writeln("Ran %d test%s in %.3fs" % - (run, run != 1 and "s" or "", timeTaken)) + (run, run != 1 and "s" or "", time_taken)) self.stream.writeln() - expectedFails = unexpectedSuccesses = skipped = 0 + expected_fails = unexpected_successes = skipped = 0 try: results = map(len, (result.expectedFailures, result.unexpectedSuccesses, @@ -264,26 +283,35 @@ def run(self, test): except AttributeError: pass else: - expectedFails, unexpectedSuccesses, skipped = results + expected_fails, unexpected_successes, skipped = results infos = [] + ansi = get_colors() + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW + if not result.wasSuccessful(): - self.stream.write("FAILED") + self.stream.write(f"{bold_red}FAILED{reset}") failed, errored = len(result.failures), len(result.errors) if failed: - infos.append("failures=%d" % failed) + infos.append(f"{bold_red}failures={failed}{reset}") if errored: - infos.append("errors=%d" % errored) + infos.append(f"{bold_red}errors={errored}{reset}") elif run == 0 and not skipped: - self.stream.write("NO TESTS RAN") + self.stream.write(f"{yellow}NO TESTS RAN{reset}") else: - self.stream.write("OK") + self.stream.write(f"{green}OK{reset}") if skipped: - infos.append("skipped=%d" % skipped) - if expectedFails: - infos.append("expected failures=%d" % expectedFails) - if unexpectedSuccesses: - infos.append("unexpected successes=%d" % unexpectedSuccesses) + infos.append(f"{yellow}skipped={skipped}{reset}") + if expected_fails: + infos.append(f"{yellow}expected failures={expected_fails}{reset}") + if unexpected_successes: + infos.append( + f"{red}unexpected successes={unexpected_successes}{reset}" + ) if infos: self.stream.writeln(" (%s)" % (", ".join(infos),)) else: diff --git a/Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst b/Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst new file mode 100644 index 00000000000000..0e4a03caf9f49d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst @@ -0,0 +1 @@ +Add colour to :mod:`unittest` output. Patch by Hugo van Kemenade. From 657d0e99aa8754372786120d6ec00c9d9970e775 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Thu, 5 Dec 2024 21:52:58 +0100 Subject: [PATCH 114/427] [Docs] GDB howto: Fix block type of a cast example (#127621) --- Doc/howto/gdb_helpers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst index 53bbf7ddaa2ab9..98ce813ca4ab02 100644 --- a/Doc/howto/gdb_helpers.rst +++ b/Doc/howto/gdb_helpers.rst @@ -180,7 +180,7 @@ regular machine-level integer:: (gdb) p some_python_integer $4 = 42 -The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`: +The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`:: (gdb) p *(PyLongObject*)some_python_integer $5 = {ob_base = {ob_base = {ob_refcnt = 8, ob_type = 0x3dad39f5e0}, ob_size = 1}, From f4f530804b9d8f089eba0f157ec2144c03b13651 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 5 Dec 2024 21:07:31 +0000 Subject: [PATCH 115/427] gh-127582: Make object resurrection thread-safe for free threading. (GH-127612) Objects may be temporarily "resurrected" in destructors when calling finalizers or watcher callbacks. We previously undid the resurrection by decrementing the reference count using `Py_SET_REFCNT`. This was not thread-safe because other threads might be accessing the object (modifying its reference count) if it was exposed by the finalizer, watcher callback, or temporarily accessed by a racy dictionary or list access. This adds internal-only thread-safe functions for temporary object resurrection during destructors. --- Include/internal/pycore_object.h | 44 +++++++++++++++++++ ...-12-05-19-25-00.gh-issue-127582.ogUY2a.rst | 2 + Objects/codeobject.c | 7 +-- Objects/dictobject.c | 7 +-- Objects/funcobject.c | 7 +-- Objects/object.c | 40 ++++++++++++++--- 6 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index ce876b093b2522..6b0b464a6fdb96 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -697,8 +697,52 @@ _PyObject_SetMaybeWeakref(PyObject *op) } } +extern int _PyObject_ResurrectEndSlow(PyObject *op); #endif +// Temporarily resurrects an object during deallocation. The refcount is set +// to one. +static inline void +_PyObject_ResurrectStart(PyObject *op) +{ + assert(Py_REFCNT(op) == 0); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif +#ifdef Py_GIL_DISABLED + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_ThreadId()); + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 1); + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0); +#else + Py_SET_REFCNT(op, 1); +#endif +} + +// Undoes an object resurrection by decrementing the refcount without calling +// _Py_Dealloc(). Returns 0 if the object is dead (the normal case), and +// deallocation should continue. Returns 1 if the object is still alive. +static inline int +_PyObject_ResurrectEnd(PyObject *op) +{ +#ifdef Py_REF_DEBUG + _Py_DecRefTotal(_PyThreadState_GET()); +#endif +#ifndef Py_GIL_DISABLED + Py_SET_REFCNT(op, Py_REFCNT(op) - 1); + return Py_REFCNT(op) != 0; +#else + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); + if (_Py_IsOwnedByCurrentThread(op) && local == 1 && shared == 0) { + // Fast-path: object has a single refcount and is owned by this thread + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); + return 0; + } + // Slow-path: object has a shared refcount or is not owned by this thread + return _PyObject_ResurrectEndSlow(op); +#endif +} + /* Tries to incref op and returns 1 if successful or 0 otherwise. */ static inline int _Py_TryIncref(PyObject *op) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst new file mode 100644 index 00000000000000..59491feeb9bcfa --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst @@ -0,0 +1,2 @@ +Fix non-thread-safe object resurrection when calling finalizers and watcher +callbacks in the free threading build. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 148350cc4b9195..eb8de136ee6432 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1867,14 +1867,11 @@ free_monitoring_data(_PyCoMonitoringData *data) static void code_dealloc(PyCodeObject *co) { - assert(Py_REFCNT(co) == 0); - Py_SET_REFCNT(co, 1); + _PyObject_ResurrectStart((PyObject *)co); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); - if (Py_REFCNT(co) > 1) { - Py_SET_REFCNT(co, Py_REFCNT(co) - 1); + if (_PyObject_ResurrectEnd((PyObject *)co)) { return; } - Py_SET_REFCNT(co, 0); #ifdef Py_GIL_DISABLED PyObject_GC_UnTrack(co); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a13d8084d14d66..1c9f86438dadc3 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3162,14 +3162,11 @@ dict_dealloc(PyObject *self) { PyDictObject *mp = (PyDictObject *)self; PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(Py_REFCNT(mp) == 0); - Py_SET_REFCNT(mp, 1); + _PyObject_ResurrectStart(self); _PyDict_NotifyEvent(interp, PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); - if (Py_REFCNT(mp) > 1) { - Py_SET_REFCNT(mp, Py_REFCNT(mp) - 1); + if (_PyObject_ResurrectEnd(self)) { return; } - Py_SET_REFCNT(mp, 0); PyDictValues *values = mp->ma_values; PyDictKeysObject *keys = mp->ma_keys; Py_ssize_t i, n; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 4ba47285f7152f..cca7f01498013e 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1092,14 +1092,11 @@ static void func_dealloc(PyObject *self) { PyFunctionObject *op = _PyFunction_CAST(self); - assert(Py_REFCNT(op) == 0); - Py_SET_REFCNT(op, 1); + _PyObject_ResurrectStart(self); handle_func_event(PyFunction_EVENT_DESTROY, op, NULL); - if (Py_REFCNT(op) > 1) { - Py_SET_REFCNT(op, Py_REFCNT(op) - 1); + if (_PyObject_ResurrectEnd(self)) { return; } - Py_SET_REFCNT(op, 0); _PyObject_GC_UNTRACK(op); if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); diff --git a/Objects/object.c b/Objects/object.c index 8868fa29066404..74f47fa4239032 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -362,8 +362,10 @@ is_dead(PyObject *o) } # endif -void -_Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) +// Decrement the shared reference count of an object. Return 1 if the object +// is dead and should be deallocated, 0 otherwise. +static int +_Py_DecRefSharedIsDead(PyObject *o, const char *filename, int lineno) { // Should we queue the object for the owning thread to merge? int should_queue; @@ -404,6 +406,15 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) } else if (new_shared == _Py_REF_MERGED) { // refcount is zero AND merged + return 1; + } + return 0; +} + +void +_Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) +{ + if (_Py_DecRefSharedIsDead(o, filename, lineno)) { _Py_Dealloc(o); } } @@ -472,6 +483,26 @@ _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) &shared, new_shared)); return refcnt; } + +// The more complicated "slow" path for undoing the resurrection of an object. +int +_PyObject_ResurrectEndSlow(PyObject *op) +{ + if (_Py_IsImmortal(op)) { + return 1; + } + if (_Py_IsOwnedByCurrentThread(op)) { + // If the object is owned by the current thread, give up ownership and + // merge the refcount. This isn't necessary in all cases, but it + // simplifies the implementation. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(op, -1); + return refcount != 0; + } + int is_dead = _Py_DecRefSharedIsDead(op, NULL, 0); + return !is_dead; +} + + #endif /* Py_GIL_DISABLED */ @@ -550,7 +581,7 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) } /* Temporarily resurrect the object. */ - Py_SET_REFCNT(self, 1); + _PyObject_ResurrectStart(self); PyObject_CallFinalizer(self); @@ -560,8 +591,7 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) /* Undo the temporary resurrection; can't use DECREF here, it would * cause a recursive call. */ - Py_SET_REFCNT(self, Py_REFCNT(self) - 1); - if (Py_REFCNT(self) == 0) { + if (!_PyObject_ResurrectEnd(self)) { return 0; /* this is the normal path out */ } From 8b3cccf3f9508572d85b0044519f2bd5715dacad Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 5 Dec 2024 21:39:43 +0000 Subject: [PATCH 116/427] GH-125413: Revert addition of `pathlib.Path.scandir()` method (#127377) Remove documentation for `pathlib.Path.scandir()`, and rename the method to `_scandir()`. In the private pathlib ABCs, make `iterdir()` abstract and call it from `_scandir()`. It's not worthwhile to add this method at the moment - see discussion: https://discuss.python.org/t/ergonomics-of-new-pathlib-path-scandir/71721 Co-authored-by: Steve Dower --- Doc/library/pathlib.rst | 29 ----------- Doc/whatsnew/3.14.rst | 6 --- Lib/pathlib/_abc.py | 15 +++--- Lib/pathlib/_local.py | 4 +- Lib/test/test_pathlib/test_pathlib_abc.py | 48 ++++--------------- Misc/NEWS.d/3.14.0a2.rst | 2 +- ...-11-29-00-15-59.gh-issue-125413.WCN0vv.rst | 3 ++ 7 files changed, 22 insertions(+), 85 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index a42ac1f8bcdf71..4b48880d6d9a18 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1289,35 +1289,6 @@ Reading directories raised. -.. method:: Path.scandir() - - When the path points to a directory, return an iterator of - :class:`os.DirEntry` objects corresponding to entries in the directory. The - returned iterator supports the :term:`context manager` protocol. It is - implemented using :func:`os.scandir` and gives the same guarantees. - - Using :meth:`~Path.scandir` instead of :meth:`~Path.iterdir` can - significantly increase the performance of code that also needs file type or - file attribute information, because :class:`os.DirEntry` objects expose - this information if the operating system provides it when scanning a - directory. - - The following example displays the names of subdirectories. The - ``entry.is_dir()`` check will generally not make an additional system call:: - - >>> p = Path('docs') - >>> with p.scandir() as entries: - ... for entry in entries: - ... if entry.is_dir(): - ... entry.name - ... - '_templates' - '_build' - '_static' - - .. versionadded:: 3.14 - - .. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False) Glob the given relative *pattern* in the directory represented by this path, diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index db25c037e509b6..b300e348679438 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -532,12 +532,6 @@ pathlib (Contributed by Barney Gale in :gh:`73991`.) -* Add :meth:`pathlib.Path.scandir` to scan a directory and return an iterator - of :class:`os.DirEntry` objects. This is exactly equivalent to calling - :func:`os.scandir` on a path object. - - (Contributed by Barney Gale in :gh:`125413`.) - pdb --- diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 2b314b6c9a16bf..86617ff2616f33 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -94,7 +94,7 @@ class PathGlobber(_GlobberBase): lexists = operator.methodcaller('exists', follow_symlinks=False) add_slash = operator.methodcaller('joinpath', '') - scandir = operator.methodcaller('scandir') + scandir = operator.methodcaller('_scandir') @staticmethod def concat_path(path, text): @@ -632,13 +632,14 @@ def write_text(self, data, encoding=None, errors=None, newline=None): with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) - def scandir(self): - """Yield os.DirEntry objects of the directory contents. + def _scandir(self): + """Yield os.DirEntry-like objects of the directory contents. The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - raise UnsupportedOperation(self._unsupported_msg('scandir()')) + import contextlib + return contextlib.nullcontext(self.iterdir()) def iterdir(self): """Yield path objects of the directory contents. @@ -646,9 +647,7 @@ def iterdir(self): The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - with self.scandir() as entries: - names = [entry.name for entry in entries] - return map(self.joinpath, names) + raise UnsupportedOperation(self._unsupported_msg('iterdir()')) def _glob_selector(self, parts, case_sensitive, recurse_symlinks): if case_sensitive is None: @@ -698,7 +697,7 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): if not top_down: paths.append((path, dirnames, filenames)) try: - with path.scandir() as entries: + with path._scandir() as entries: for entry in entries: name = entry.name try: diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index b5d9dc49f58463..bb8a252c0e94e2 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -634,8 +634,8 @@ def _filter_trailing_slash(self, paths): path_str = path_str[:-1] yield path_str - def scandir(self): - """Yield os.DirEntry objects of the directory contents. + def _scandir(self): + """Yield os.DirEntry-like objects of the directory contents. The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 5fa2f550cefcf4..7ba3fa823a30b9 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1,5 +1,4 @@ import collections -import contextlib import io import os import errno @@ -1418,24 +1417,6 @@ def close(self): 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') -class DummyDirEntry: - """ - Minimal os.DirEntry-like object. Returned from DummyPath.scandir(). - """ - __slots__ = ('name', '_is_symlink', '_is_dir') - - def __init__(self, name, is_symlink, is_dir): - self.name = name - self._is_symlink = is_symlink - self._is_dir = is_dir - - def is_symlink(self): - return self._is_symlink - - def is_dir(self, *, follow_symlinks=True): - return self._is_dir and (follow_symlinks or not self._is_symlink) - - class DummyPath(PathBase): """ Simple implementation of PathBase that keeps files and directories in @@ -1503,25 +1484,14 @@ def open(self, mode='r', buffering=-1, encoding=None, stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) return stream - @contextlib.contextmanager - def scandir(self): - path = self.resolve() - path_str = str(path) - if path_str in self._files: - raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path_str) - elif path_str in self._directories: - yield iter([path.joinpath(name)._dir_entry for name in self._directories[path_str]]) + def iterdir(self): + path = str(self.resolve()) + if path in self._files: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) + elif path in self._directories: + return iter([self / name for name in self._directories[path]]) else: - raise FileNotFoundError(errno.ENOENT, "File not found", path_str) - - @property - def _dir_entry(self): - path_str = str(self) - is_symlink = path_str in self._symlinks - is_directory = (path_str in self._directories - if not is_symlink - else self._symlinks[path_str][1]) - return DummyDirEntry(self.name, is_symlink, is_directory) + raise FileNotFoundError(errno.ENOENT, "File not found", path) def mkdir(self, mode=0o777, parents=False, exist_ok=False): path = str(self.parent.resolve() / self.name) @@ -2214,9 +2184,9 @@ def test_iterdir_nodir(self): def test_scandir(self): p = self.cls(self.base) - with p.scandir() as entries: + with p._scandir() as entries: self.assertTrue(list(entries)) - with p.scandir() as entries: + with p._scandir() as entries: for entry in entries: child = p / entry.name self.assertIsNotNone(entry) diff --git a/Misc/NEWS.d/3.14.0a2.rst b/Misc/NEWS.d/3.14.0a2.rst index 7384ce54cb8914..d82ec98b7a3c87 100644 --- a/Misc/NEWS.d/3.14.0a2.rst +++ b/Misc/NEWS.d/3.14.0a2.rst @@ -597,7 +597,7 @@ TypeError is now raised instead of ValueError for some logical errors. .. nonce: Jat5kq .. section: Library -Add :meth:`pathlib.Path.scandir` method to efficiently fetch directory +Add :meth:`!pathlib.Path.scandir` method to efficiently fetch directory children and their file attributes. This is a trivial wrapper of :func:`os.scandir`. diff --git a/Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst b/Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst new file mode 100644 index 00000000000000..b56a77b4294ace --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst @@ -0,0 +1,3 @@ +Revert addition of :meth:`!pathlib.Path.scandir`. This method was added in +3.14.0a2. The optimizations remain for file system paths, but other +subclasses should only have to implement :meth:`pathlib.Path.iterdir`. From 25eee578c8e369b027da6d9d2725f29df6ef1cbd Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 6 Dec 2024 03:47:51 +0100 Subject: [PATCH 117/427] gh-127627: Add `posix._emscripten_debugger` function (#127628) Add a posix._emscripten_debugger function to add an emscripten breakpoint. --- ...-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst | 2 ++ Modules/clinic/posixmodule.c.h | 28 ++++++++++++++++++- Modules/posixmodule.c | 22 ++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst diff --git a/Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst b/Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst new file mode 100644 index 00000000000000..48a6c7d30b4a26 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst @@ -0,0 +1,2 @@ +Added ``posix._emscripten_debugger()`` to help with debugging the test suite on +the Emscripten target. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index cd0c4faeac83d1..554299b8598299 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -12447,6 +12447,28 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) return os__create_environ_impl(module); } +#if defined(__EMSCRIPTEN__) + +PyDoc_STRVAR(os__emscripten_debugger__doc__, +"_emscripten_debugger($module, /)\n" +"--\n" +"\n" +"Create a breakpoint for the JavaScript debugger. Emscripten only."); + +#define OS__EMSCRIPTEN_DEBUGGER_METHODDEF \ + {"_emscripten_debugger", (PyCFunction)os__emscripten_debugger, METH_NOARGS, os__emscripten_debugger__doc__}, + +static PyObject * +os__emscripten_debugger_impl(PyObject *module); + +static PyObject * +os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__emscripten_debugger_impl(module); +} + +#endif /* defined(__EMSCRIPTEN__) */ + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -13114,4 +13136,8 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=7ee14f5e880092f5 input=a9049054013a1b77]*/ + +#ifndef OS__EMSCRIPTEN_DEBUGGER_METHODDEF + #define OS__EMSCRIPTEN_DEBUGGER_METHODDEF +#endif /* !defined(OS__EMSCRIPTEN_DEBUGGER_METHODDEF) */ +/*[clinic end generated code: output=9c2ca1dbf986c62c input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2c26fbeac9a1be..2045c6065b8e7a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -84,6 +84,9 @@ extern char * _getpty(int *, int, mode_t, int); #endif +#ifdef __EMSCRIPTEN__ +#include "emscripten.h" // emscripten_debugger() +#endif /* * A number of APIs are available on macOS from a certain macOS version. @@ -16845,8 +16848,24 @@ os__create_environ_impl(PyObject *module) } -static PyMethodDef posix_methods[] = { +#ifdef __EMSCRIPTEN__ +/*[clinic input] +os._emscripten_debugger + +Create a breakpoint for the JavaScript debugger. Emscripten only. +[clinic start generated code]*/ + +static PyObject * +os__emscripten_debugger_impl(PyObject *module) +/*[clinic end generated code: output=ad47dc3bf0661343 input=d814b1877fb6083a]*/ +{ + emscripten_debugger(); + Py_RETURN_NONE; +} +#endif /* __EMSCRIPTEN__ */ + +static PyMethodDef posix_methods[] = { OS_STAT_METHODDEF OS_ACCESS_METHODDEF OS_TTYNAME_METHODDEF @@ -17060,6 +17079,7 @@ static PyMethodDef posix_methods[] = { OS__INPUTHOOK_METHODDEF OS__IS_INPUTHOOK_INSTALLED_METHODDEF OS__CREATE_ENVIRON_METHODDEF + OS__EMSCRIPTEN_DEBUGGER_METHODDEF {NULL, NULL} /* Sentinel */ }; From e991ac8f2037d78140e417cc9a9486223eb3e786 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 5 Dec 2024 22:33:03 -0600 Subject: [PATCH 118/427] gh-127655: Ensure `_SelectorSocketTransport.writelines` pauses the protocol if needed (#127656) Ensure `_SelectorSocketTransport.writelines` pauses the protocol if it reaches the high water mark as needed. Co-authored-by: Kumar Aditya --- Lib/asyncio/selector_events.py | 1 + Lib/test/test_asyncio/test_selector_events.py | 12 ++++++++++++ .../2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst | 1 + 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index f94bf10b4225e7..f1ab9b12d69a5d 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1175,6 +1175,7 @@ def writelines(self, list_of_data): # If the entire buffer couldn't be written, register a write handler if self._buffer: self._loop._add_writer(self._sock_fd, self._write_ready) + self._maybe_pause_protocol() def can_write_eof(self): return True diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index aaeda33dd0c677..efca30f37414f9 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -805,6 +805,18 @@ def test_writelines_send_partial(self): self.assertTrue(self.sock.send.called) self.assertTrue(self.loop.writers) + def test_writelines_pauses_protocol(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + self.sock.send.fileno.return_value = 7 + + transport = self.socket_transport() + transport._high_water = 1 + transport.writelines([data]) + self.assertTrue(self.protocol.pause_writing.called) + self.assertTrue(self.sock.send.called) + self.assertTrue(self.loop.writers) + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') def test_write_sendmsg_full(self): data = memoryview(b'data') diff --git a/Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst b/Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst new file mode 100644 index 00000000000000..76cfc58121d3bd --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst @@ -0,0 +1 @@ +Fixed the :class:`!asyncio.selector_events._SelectorSocketTransport` transport not pausing writes for the protocol when the buffer reaches the high water mark when using :meth:`asyncio.WriteTransport.writelines`. From 8b7c194c7bf7e547e4f6317528f0dcb9344c18c7 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Fri, 6 Dec 2024 13:28:32 +0300 Subject: [PATCH 119/427] gh-120010: Fix invalid (nan+nanj) results in _Py_c_prod() (GH-120287) In some cases, previously computed as (nan+nanj), we could recover meaningful component values in the result, see e.g. the C11, Annex G.5.1, routine _Cmultd(): >>> z = 1e300+1j >>> z*(nan+infj) # was (nan+nanj) (-inf+infj) That also fix some complex powers for small integer exponents, computed with optimized algorithm (by squaring): >>> z**5 # was (nan+nanj) Traceback (most recent call last): File "", line 1, in z**5 ~^^~ OverflowError: complex exponentiation --- Lib/test/test_complex.py | 17 ++++++ ...-06-04-08-26-25.gh-issue-120010._z-AWz.rst | 2 + Objects/complexobject.c | 60 +++++++++++++++++-- 3 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 179556f57e884f..fd002fb00ac338 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -299,6 +299,22 @@ def test_mul(self): self.assertRaises(TypeError, operator.mul, 1j, None) self.assertRaises(TypeError, operator.mul, None, 1j) + for z, w, r in [(1e300+1j, complex(INF, INF), complex(NAN, INF)), + (1e300+1j, complex(NAN, INF), complex(-INF, INF)), + (1e300+1j, complex(INF, NAN), complex(INF, INF)), + (complex(INF, 1), complex(NAN, INF), complex(NAN, INF)), + (complex(INF, 1), complex(INF, NAN), complex(INF, NAN)), + (complex(NAN, 1), complex(1, INF), complex(-INF, NAN)), + (complex(1, NAN), complex(1, INF), complex(NAN, INF)), + (complex(1e200, NAN), complex(1e200, NAN), complex(INF, NAN)), + (complex(1e200, NAN), complex(NAN, 1e200), complex(NAN, INF)), + (complex(NAN, 1e200), complex(1e200, NAN), complex(NAN, INF)), + (complex(NAN, 1e200), complex(NAN, 1e200), complex(-INF, NAN)), + (complex(NAN, NAN), complex(NAN, NAN), complex(NAN, NAN))]: + with self.subTest(z=z, w=w, r=r): + self.assertComplexesAreIdentical(z * w, r) + self.assertComplexesAreIdentical(w * z, r) + def test_mod(self): # % is no longer supported on complex numbers with self.assertRaises(TypeError): @@ -340,6 +356,7 @@ def test_pow(self): self.assertAlmostEqual(pow(1j, 200), 1) self.assertRaises(ValueError, pow, 1+1j, 1+1j, 1+1j) self.assertRaises(OverflowError, pow, 1e200+1j, 1e200+1j) + self.assertRaises(OverflowError, pow, 1e200+1j, 5) self.assertRaises(TypeError, pow, 1j, None) self.assertRaises(TypeError, pow, None, 1j) self.assertAlmostEqual(pow(1j, 0.5), 0.7071067811865476+0.7071067811865475j) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst new file mode 100644 index 00000000000000..7954c7f5927397 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst @@ -0,0 +1,2 @@ +Correct invalid corner cases which resulted in ``(nan+nanj)`` output in complex +multiplication, e.g., ``(1e300+1j)*(nan+infj)``. Patch by Sergey B Kirpichev. diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 8fbca3cb02d80a..bf6187efac941f 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -85,11 +85,63 @@ _Py_c_neg(Py_complex a) } Py_complex -_Py_c_prod(Py_complex a, Py_complex b) +_Py_c_prod(Py_complex z, Py_complex w) { - Py_complex r; - r.real = a.real*b.real - a.imag*b.imag; - r.imag = a.real*b.imag + a.imag*b.real; + double a = z.real, b = z.imag, c = w.real, d = w.imag; + double ac = a*c, bd = b*d, ad = a*d, bc = b*c; + Py_complex r = {ac - bd, ad + bc}; + + /* Recover infinities that computed as nan+nanj. See e.g. the C11, + Annex G.5.1, routine _Cmultd(). */ + if (isnan(r.real) && isnan(r.imag)) { + int recalc = 0; + + if (isinf(a) || isinf(b)) { /* z is infinite */ + /* "Box" the infinity and change nans in the other factor to 0 */ + a = copysign(isinf(a) ? 1.0 : 0.0, a); + b = copysign(isinf(b) ? 1.0 : 0.0, b); + if (isnan(c)) { + c = copysign(0.0, c); + } + if (isnan(d)) { + d = copysign(0.0, d); + } + recalc = 1; + } + if (isinf(c) || isinf(d)) { /* w is infinite */ + /* "Box" the infinity and change nans in the other factor to 0 */ + c = copysign(isinf(c) ? 1.0 : 0.0, c); + d = copysign(isinf(d) ? 1.0 : 0.0, d); + if (isnan(a)) { + a = copysign(0.0, a); + } + if (isnan(b)) { + b = copysign(0.0, b); + } + recalc = 1; + } + if (!recalc && (isinf(ac) || isinf(bd) || isinf(ad) || isinf(bc))) { + /* Recover infinities from overflow by changing nans to 0 */ + if (isnan(a)) { + a = copysign(0.0, a); + } + if (isnan(b)) { + b = copysign(0.0, b); + } + if (isnan(c)) { + c = copysign(0.0, c); + } + if (isnan(d)) { + d = copysign(0.0, d); + } + recalc = 1; + } + if (recalc) { + r.real = Py_INFINITY*(a*c - b*d); + r.imag = Py_INFINITY*(a*d + b*c); + } + } + return r; } From 023b7d2141467017abc27de864f3f44677768cb3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 6 Dec 2024 10:46:59 +0000 Subject: [PATCH 120/427] GH-126491: Lower heap size limit with faster marking (GH-127519) * Faster marking of reachable objects * Changes calculation of work to do and work done. * Merges transitive closure calculations --- InternalDocs/garbage_collector.md | 50 ++++- Lib/test/test_gc.py | 14 +- Objects/dictobject.c | 4 +- Objects/genobject.c | 69 +------ Objects/typeobject.c | 13 ++ Python/gc.c | 301 ++++++++++++++---------------- 6 files changed, 208 insertions(+), 243 deletions(-) diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 08db080a200ea4..4761f78f3593e3 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -199,22 +199,22 @@ unreachable: ```pycon >>> import gc ->>> +>>> >>> class Link: ... def __init__(self, next_link=None): ... self.next_link = next_link -... +... >>> link_3 = Link() >>> link_2 = Link(link_3) >>> link_1 = Link(link_2) >>> link_3.next_link = link_1 >>> A = link_1 >>> del link_1, link_2, link_3 ->>> +>>> >>> link_4 = Link() >>> link_4.next_link = link_4 >>> del link_4 ->>> +>>> >>> # Collect the unreachable Link object (and its .__dict__ dict). >>> gc.collect() 2 @@ -459,11 +459,11 @@ specifically in a generation by calling `gc.collect(generation=NUM)`. >>> # Create a reference cycle. >>> x = MyObj() >>> x.self = x ->>> +>>> >>> # Initially the object is in the young generation. >>> gc.get_objects(generation=0) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ->>> +>>> >>> # After a collection of the youngest generation the object >>> # moves to the old generation. >>> gc.collect(generation=0) @@ -515,6 +515,44 @@ increment. All objects directly referred to from those stack frames are added to the working set. Then the above algorithm is repeated, starting from step 2. +Determining how much work to do +------------------------------- + +We need to do a certain amount of work to enusre that garbage is collected, +but doing too much work slows down execution. + +To work out how much work we need to do, consider a heap with `L` live objects +and `G0` garbage objects at the start of a full scavenge and `G1` garbage objects +at the end of the scavenge. We don't want the amount of garbage to grow, `G1 ≤ G0`, and +we don't want too much garbage (say 1/3 of the heap maximum), `G0 ≤ L/2`. +For each full scavenge we must visit all objects, `T == L + G0 + G1`, during which +`G1` garbage objects are created. + +The number of new objects created `N` must be at least the new garbage created, `N ≥ G1`, +assuming that the number of live objects remains roughly constant. +If we set `T == 4*N` we get `T > 4*G1` and `T = L + G0 + G1` => `L + G0 > 3G1` +For a steady state heap (`G0 == G1`) we get `L > 2G0` and the desired garbage ratio. + +In other words, to keep the garbage fraction to 1/3 or less we need to visit +4 times as many objects as are newly created. + +We can do better than this though. Not all new objects will be garbage. +Consider the heap at the end of the scavenge with `L1` live objects and `G1` +garbage. Also, note that `T == M + I` where `M` is the number of objects marked +as reachable and `I` is the number of objects visited in increments. +Everything in `M` is live, so `I ≥ G0` and in practice `I` is closer to `G0 + G1`. + +If we choose the amount of work done such that `2*M + I == 6N` then we can do +less work in most cases, but are still guaranteed to keep up. +Since `I ≳ G0 + G1` (not strictly true, but close enough) +`T == M + I == (6N + I)/2` and `(6N + I)/2 ≳ 4G`, so we can keep up. + +The reason that this improves performance is that `M` is usually much larger +than `I`. If `M == 10I`, then `T ≅ 3N`. + +Finally, instead of using a fixed multiple of 8, we gradually increase it as the +heap grows. This avoids wasting work for small heaps and during startup. + Optimization: reusing fields to save memory =========================================== diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index b5140057a69d36..baf8e95dffdfce 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1161,27 +1161,19 @@ def make_ll(depth): return head head = make_ll(1000) - count = 1000 - - # There will be some objects we aren't counting, - # e.g. the gc stats dicts. This test checks - # that the counts don't grow, so we try to - # correct for the uncounted objects - # This is just an estimate. - CORRECTION = 20 enabled = gc.isenabled() gc.enable() olds = [] initial_heap_size = _testinternalcapi.get_tracked_heap_size() - for i in range(20_000): + iterations = max(20_000, initial_heap_size) + for i in range(iterations): newhead = make_ll(20) - count += 20 newhead.surprise = head olds.append(newhead) if len(olds) == 20: new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size - self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations") + self.assertLess(new_objects, initial_heap_size/2, f"Heap growing. Reached limit after {i} iterations") del olds[:] if not enabled: gc.disable() diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 1c9f86438dadc3..de518b8dc5024b 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -7064,9 +7064,7 @@ int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) { PyTypeObject *tp = Py_TYPE(obj); - if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - return 0; - } + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictValues *values = _PyObject_InlineValues(obj); if (values->valid) { diff --git a/Objects/genobject.c b/Objects/genobject.c index e87f199c2504ba..33679afecb420f 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -882,25 +882,7 @@ PyTypeObject PyGen_Type = { gen_methods, /* tp_methods */ gen_memberlist, /* tp_members */ gen_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - _PyGen_Finalize, /* tp_finalize */ + .tp_finalize = _PyGen_Finalize, }; static PyObject * @@ -1242,24 +1224,7 @@ PyTypeObject PyCoro_Type = { coro_methods, /* tp_methods */ coro_memberlist, /* tp_members */ coro_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - _PyGen_Finalize, /* tp_finalize */ + .tp_finalize = _PyGen_Finalize, }; static void @@ -1464,7 +1429,6 @@ typedef struct _PyAsyncGenWrappedValue { (assert(_PyAsyncGenWrappedValue_CheckExact(op)), \ _Py_CAST(_PyAsyncGenWrappedValue*, (op))) - static int async_gen_traverse(PyObject *self, visitproc visit, void *arg) { @@ -1673,24 +1637,7 @@ PyTypeObject PyAsyncGen_Type = { async_gen_methods, /* tp_methods */ async_gen_memberlist, /* tp_members */ async_gen_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - _PyGen_Finalize, /* tp_finalize */ + .tp_finalize = _PyGen_Finalize, }; @@ -1935,16 +1882,6 @@ PyTypeObject _PyAsyncGenASend_Type = { PyObject_SelfIter, /* tp_iter */ async_gen_asend_iternext, /* tp_iternext */ async_gen_asend_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ .tp_finalize = async_gen_asend_finalize, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2068d6aa9be52b..cc95b9857e3f2d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2355,6 +2355,16 @@ subtype_traverse(PyObject *self, visitproc visit, void *arg) return 0; } + +static int +plain_object_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyTypeObject *type = Py_TYPE(self); + assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); + Py_VISIT(type); + return PyObject_VisitManagedDict(self, visit, arg); +} + static void clear_slots(PyTypeObject *type, PyObject *self) { @@ -4147,6 +4157,9 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) assert((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); type->tp_flags |= Py_TPFLAGS_MANAGED_DICT; type->tp_dictoffset = -1; + if (type->tp_basicsize == sizeof(PyObject)) { + type->tp_traverse = plain_object_traverse; + } } type->tp_basicsize = slotoffset; diff --git a/Python/gc.c b/Python/gc.c index 5b9588c8741b97..fd29a48518e71b 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1277,18 +1277,13 @@ gc_list_set_space(PyGC_Head *list, int space) * space faster than objects are added to the old space. * * Each young or incremental collection adds a number of - * objects, S (for survivors) to the old space, and - * incremental collectors scan I objects from the old space. - * I > S must be true. We also want I > S * N to be where - * N > 1. Higher values of N mean that the old space is + * new objects (N) to the heap, and incremental collectors + * scan I objects from the old space. + * I > N must be true. We also want I > N * K to be where + * K > 2. Higher values of K mean that the old space is * scanned more rapidly. - * The default incremental threshold of 10 translates to - * N == 1.4 (1 + 4/threshold) */ - -/* Divide by 10, so that the default incremental threshold of 10 - * scans objects at 1% of the heap size */ -#define SCAN_RATE_DIVISOR 10 +#define SCAN_RATE_DIVISOR 5 static void add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats) @@ -1330,69 +1325,76 @@ gc_collect_young(PyThreadState *tstate, validate_spaces(gcstate); } -#ifndef NDEBUG -static inline int -IS_IN_VISITED(PyGC_Head *gc, int visited_space) +typedef struct work_stack { + PyGC_Head *top; + int visited_space; +} WorkStack; + +/* Remove gc from the list it is currently in and push it to the stack */ +static inline void +push_to_stack(PyGC_Head *gc, WorkStack *stack) { - assert(visited_space == 0 || other_space(visited_space) == 0); - return gc_old_space(gc) == visited_space; + PyGC_Head *prev = GC_PREV(gc); + PyGC_Head *next = GC_NEXT(gc); + _PyGCHead_SET_NEXT(prev, next); + _PyGCHead_SET_PREV(next, prev); + _PyGCHead_SET_PREV(gc, stack->top); + stack->top = gc; } -#endif -struct container_and_flag { - PyGC_Head *container; - int visited_space; - intptr_t size; -}; +static inline PyGC_Head * +pop_from_stack(WorkStack *stack) +{ + PyGC_Head *gc = stack->top; + stack->top = _PyGCHead_PREV(gc); + return gc; +} -/* A traversal callback for adding to container) */ -static int -visit_add_to_container(PyObject *op, void *arg) +/* append list `from` to `stack`; `from` becomes an empty list */ +static void +move_list_to_stack(PyGC_Head *from, WorkStack *stack) { - OBJECT_STAT_INC(object_visits); - struct container_and_flag *cf = (struct container_and_flag *)arg; - int visited = cf->visited_space; - assert(visited == get_gc_state()->visited_space); - if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + if (!gc_list_is_empty(from)) { + PyGC_Head *from_head = GC_NEXT(from); + PyGC_Head *from_tail = GC_PREV(from); + _PyGCHead_SET_PREV(from_head, stack->top); + stack->top = from_tail; + gc_list_init(from); + } +} + +static inline void +move_to_stack(PyObject *op, WorkStack *stack, int visited_space) +{ + assert(op != NULL); + if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op) && - gc_old_space(gc) != visited) { + gc_old_space(gc) != visited_space) { + assert(!_Py_IsImmortal(op)); gc_flip_old_space(gc); - gc_list_move(gc, cf->container); - cf->size++; + push_to_stack(gc, stack); } } - return 0; } -static intptr_t -expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate) -{ - struct container_and_flag arg = { - .container = container, - .visited_space = gcstate->visited_space, - .size = 0 - }; - assert(GC_NEXT(gc) == container); - while (gc != container) { - /* Survivors will be moved to visited space, so they should - * have been marked as visited */ - assert(IS_IN_VISITED(gc, gcstate->visited_space)); - PyObject *op = FROM_GC(gc); - assert(_PyObject_GC_IS_TRACKED(op)); - if (_Py_IsImmortal(op)) { - PyGC_Head *next = GC_NEXT(gc); - gc_list_move(gc, &get_gc_state()->permanent_generation.head); - gc = next; - continue; - } - traverseproc traverse = Py_TYPE(op)->tp_traverse; - (void) traverse(op, - visit_add_to_container, - &arg); - gc = GC_NEXT(gc); - } - return arg.size; +static void +move_unvisited(PyObject *op, WorkStack *stack, int visited_space) +{ + move_to_stack(op, stack, visited_space); +} + +#define MOVE_UNVISITED(O, T, V) if ((O) != NULL) move_unvisited((O), (T), (V)) + +/* A traversal callback for adding to container */ +static int +visit_add_to_container(PyObject *op, void *arg) +{ + OBJECT_STAT_INC(object_visits); + WorkStack *stack = (WorkStack *)arg; + assert(stack->visited_space == get_gc_state()->visited_space); + move_to_stack(op, stack, stack->visited_space); + return 0; } /* Do bookkeeping for a completed GC cycle */ @@ -1420,54 +1422,62 @@ completed_scavenge(GCState *gcstate) gc_list_set_space(&gcstate->old[not_visited].head, not_visited); } assert(gc_list_is_empty(&gcstate->old[visited].head)); - gcstate->work_to_do = 0; gcstate->phase = GC_PHASE_MARK; } -static intptr_t -move_to_reachable(PyObject *op, PyGC_Head *reachable, int visited_space) -{ - if (op != NULL && !_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op) && - gc_old_space(gc) != visited_space) { - gc_flip_old_space(gc); - gc_list_move(gc, reachable); - return 1; +static void +frame_move_unvisited(_PyInterpreterFrame *frame, WorkStack *stack, int visited_space) +{ + _PyStackRef *locals = frame->localsplus; + _PyStackRef *sp = frame->stackpointer; + if (frame->f_locals != NULL) { + move_unvisited(frame->f_locals, stack, visited_space); + } + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + move_unvisited(func, stack, visited_space); + while (sp > locals) { + sp--; + _PyStackRef ref = *sp; + if (!PyStackRef_IsNull(ref)) { + PyObject *op = PyStackRef_AsPyObjectBorrow(ref); + if (!_Py_IsImmortal(op)) { + move_unvisited(op, stack, visited_space); + } } } - return 0; } -static intptr_t -mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space) +static Py_ssize_t +move_all_transitively_reachable(WorkStack *stack, PyGC_Head *visited, int visited_space) { // Transitively traverse all objects from reachable, until empty - struct container_and_flag arg = { - .container = reachable, - .visited_space = visited_space, - .size = 0 - }; - while (!gc_list_is_empty(reachable)) { - PyGC_Head *gc = _PyGCHead_NEXT(reachable); + Py_ssize_t objects_marked = 0; + while (stack->top != NULL) { + PyGC_Head *gc = pop_from_stack(stack); assert(gc_old_space(gc) == visited_space); - gc_list_move(gc, visited); + gc_list_append(gc, visited); + objects_marked++; PyObject *op = FROM_GC(gc); - traverseproc traverse = Py_TYPE(op)->tp_traverse; - (void) traverse(op, - visit_add_to_container, - &arg); + assert(PyObject_IS_GC(op)); + assert(_PyObject_GC_IS_TRACKED(op)); + if (_Py_IsImmortal(op)) { + _PyObject_GC_UNTRACK(op); + } + else { + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, visit_add_to_container, stack); + } } gc_list_validate_space(visited, visited_space); - return arg.size; + return objects_marked; } static intptr_t mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start) { - PyGC_Head reachable; - gc_list_init(&reachable); - Py_ssize_t objects_marked = 0; + WorkStack stack; + stack.top = NULL; + stack.visited_space = visited_space; // Move all objects on stacks to reachable _PyRuntimeState *runtime = &_PyRuntime; HEAD_LOCK(runtime); @@ -1480,27 +1490,7 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b frame = frame->previous; continue; } - _PyStackRef *locals = frame->localsplus; - _PyStackRef *sp = frame->stackpointer; - objects_marked += move_to_reachable(frame->f_locals, &reachable, visited_space); - PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - objects_marked += move_to_reachable(func, &reachable, visited_space); - while (sp > locals) { - sp--; - if (PyStackRef_IsNull(*sp)) { - continue; - } - PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); - if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op) && - gc_old_space(gc) != visited_space) { - gc_flip_old_space(gc); - objects_marked++; - gc_list_move(gc, &reachable); - } - } - } + frame_move_unvisited(frame, &stack, visited_space); if (!start && frame->visited) { // If this frame has already been visited, then the lower frames // will have already been visited and will not have changed @@ -1513,31 +1503,31 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b ts = PyThreadState_Next(ts); HEAD_UNLOCK(runtime); } - objects_marked += mark_all_reachable(&reachable, visited, visited_space); - assert(gc_list_is_empty(&reachable)); + Py_ssize_t objects_marked = move_all_transitively_reachable(&stack, visited, visited_space); + assert(stack.top == NULL); return objects_marked; } static intptr_t mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space) { - PyGC_Head reachable; - gc_list_init(&reachable); - Py_ssize_t objects_marked = 0; - objects_marked += move_to_reachable(interp->sysdict, &reachable, visited_space); - objects_marked += move_to_reachable(interp->builtins, &reachable, visited_space); - objects_marked += move_to_reachable(interp->dict, &reachable, visited_space); + WorkStack stack; + stack.top = NULL; + stack.visited_space = visited_space; + MOVE_UNVISITED(interp->sysdict, &stack, visited_space); + MOVE_UNVISITED(interp->builtins, &stack, visited_space); + MOVE_UNVISITED(interp->dict, &stack, visited_space); struct types_state *types = &interp->types; for (int i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { - objects_marked += move_to_reachable(types->builtins.initialized[i].tp_dict, &reachable, visited_space); - objects_marked += move_to_reachable(types->builtins.initialized[i].tp_subclasses, &reachable, visited_space); + MOVE_UNVISITED(types->builtins.initialized[i].tp_dict, &stack, visited_space); + MOVE_UNVISITED(types->builtins.initialized[i].tp_subclasses, &stack, visited_space); } for (int i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { - objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_dict, &reachable, visited_space); - objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_subclasses, &reachable, visited_space); + MOVE_UNVISITED(types->for_extensions.initialized[i].tp_dict, &stack, visited_space); + MOVE_UNVISITED(types->for_extensions.initialized[i].tp_subclasses, &stack, visited_space); } - objects_marked += mark_all_reachable(&reachable, visited, visited_space); - assert(gc_list_is_empty(&reachable)); + Py_ssize_t objects_marked = move_all_transitively_reachable(&stack, visited, visited_space); + assert(stack.top == NULL); return objects_marked; } @@ -1549,39 +1539,35 @@ mark_at_start(PyThreadState *tstate) PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space); objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true); - gcstate->work_to_do -= objects_marked; gcstate->phase = GC_PHASE_COLLECT; validate_spaces(gcstate); return objects_marked; } + +/* See InternalDocs/garbage_collector.md for more details. */ +#define MAX_HEAP_PORTION_MULTIPLIER 5 +#define MARKING_PROGRESS_MULTIPLIER 2 + static intptr_t assess_work_to_do(GCState *gcstate) { - /* The amount of work we want to do depends on three things. + /* The amount of work we want to do depends on two things. * 1. The number of new objects created - * 2. The growth in heap size since the last collection - * 3. The heap size (up to the number of new objects, to avoid quadratic effects) - * - * For a steady state heap, the amount of work to do is three times the number - * of new objects added to the heap. This ensures that we stay ahead in the - * worst case of all new objects being garbage. - * - * This could be improved by tracking survival rates, but it is still a - * large improvement on the non-marking approach. + * 2. The heap size (up to a multiple of the number of new objects, to avoid quadratic effects) */ intptr_t scale_factor = gcstate->old[0].threshold; if (scale_factor < 2) { scale_factor = 2; } intptr_t new_objects = gcstate->young.count; - intptr_t max_heap_fraction = new_objects*3/2; - intptr_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; - if (heap_fraction > max_heap_fraction) { - heap_fraction = max_heap_fraction; + intptr_t max_heap_portion = new_objects * MAX_HEAP_PORTION_MULTIPLIER; + intptr_t heap_portion = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; + if (heap_portion > max_heap_portion) { + heap_portion = max_heap_portion; } gcstate->young.count = 0; - return new_objects + heap_fraction; + return new_objects + heap_portion; } static void @@ -1594,36 +1580,37 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) if (gcstate->phase == GC_PHASE_MARK) { Py_ssize_t objects_marked = mark_at_start(tstate); GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); - gcstate->work_to_do -= objects_marked; + gcstate->work_to_do -= objects_marked * MARKING_PROGRESS_MULTIPLIER; validate_spaces(gcstate); return; } PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - PyGC_Head increment; - gc_list_init(&increment); - int scale_factor = gcstate->old[0].threshold; - if (scale_factor < 2) { - scale_factor = 2; - } intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false); GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); - gcstate->work_to_do -= objects_marked; + gcstate->work_to_do -= objects_marked * MARKING_PROGRESS_MULTIPLIER; gc_list_set_space(&gcstate->young.head, gcstate->visited_space); - gc_list_merge(&gcstate->young.head, &increment); + PyGC_Head increment; + gc_list_init(&increment); + WorkStack working; + working.top = 0; + working.visited_space = gcstate->visited_space; + move_list_to_stack(&gcstate->young.head, &working); + Py_ssize_t increment_size = move_all_transitively_reachable(&working, &increment, gcstate->visited_space); gc_list_validate_space(&increment, gcstate->visited_space); - Py_ssize_t increment_size = gc_list_size(&increment); + assert(working.top == NULL); while (increment_size < gcstate->work_to_do) { if (gc_list_is_empty(not_visited)) { break; } PyGC_Head *gc = _PyGCHead_NEXT(not_visited); - gc_list_move(gc, &increment); - increment_size++; - assert(!_Py_IsImmortal(FROM_GC(gc))); gc_set_old_space(gc, gcstate->visited_space); - increment_size += expand_region_transitively_reachable(&increment, gc, gcstate); + push_to_stack(gc, &working); + assert(!_Py_IsImmortal(FROM_GC(gc))); + increment_size += move_all_transitively_reachable(&working, &increment, gcstate->visited_space); + assert(working.top == NULL); } + assert(increment_size == gc_list_size(&increment)); GC_STAT_ADD(1, objects_not_transitively_reachable, increment_size); validate_list(&increment, collecting_clear_unreachable_clear); gc_list_validate_space(&increment, gcstate->visited_space); @@ -1632,7 +1619,6 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) gc_collect_region(tstate, &increment, &survivors, stats); gc_list_merge(&survivors, visited); assert(gc_list_is_empty(&increment)); - gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; gcstate->work_to_do -= increment_size; add_stats(gcstate, 1, stats); @@ -1668,6 +1654,7 @@ gc_collect_full(PyThreadState *tstate, gcstate->old[0].count = 0; gcstate->old[1].count = 0; completed_scavenge(gcstate); + gcstate->work_to_do = -gcstate->young.threshold; _PyGC_ClearAllFreeLists(tstate->interp); validate_spaces(gcstate); add_stats(gcstate, 2, stats); From 77a61c0465c27c1c4ba7cddf4638d9ed75259671 Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Fri, 6 Dec 2024 23:09:20 +0900 Subject: [PATCH 121/427] gh-101100: amend references starting with `!~` in gh-127054 (#127684) --- Doc/tutorial/datastructures.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 263b0c2e2815a1..cbe780e075baf5 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -142,8 +142,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`!~list.append`. To retrieve an item from the -top of the stack, use :meth:`!~list.pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`!append`. To retrieve an item from the +top of the stack, use :meth:`!pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -340,7 +340,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`!~list.pop` method +value: the :keyword:`del` statement. This differs from the :meth:`!pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -500,8 +500,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`!~list.append` and -:meth:`!~list.extend`. +assignments, slice assignments, or methods like :meth:`!append` and +:meth:`!extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of From 36c6178d372b075e9c74b786cfb5e47702976b1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:31:30 +0100 Subject: [PATCH 122/427] gh-126024: fix UBSan failure in `unicodeobject.c:find_first_nonascii` (GH-127566) --- Objects/unicodeobject.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 463da06445984b..33c4747bbef488 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5083,12 +5083,9 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) const unsigned char *p2 = _Py_ALIGN_UP(p, SIZEOF_SIZE_T); #if PY_LITTLE_ENDIAN && HAVE_CTZ if (p < p2) { -#if defined(_M_AMD64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) - // x86 and amd64 are little endian and can load unaligned memory. - size_t u = *(const size_t*)p & ASCII_CHAR_MASK; -#else - size_t u = load_unaligned(p, p2 - p) & ASCII_CHAR_MASK; -#endif + size_t u; + memcpy(&u, p, sizeof(size_t)); + u &= ASCII_CHAR_MASK; if (u) { return (ctz(u) - 7) / 8; } From a353455fca1b8f468ff3ffbb4b5e316510b4fd43 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 6 Dec 2024 15:48:24 +0000 Subject: [PATCH 123/427] gh-125610: Fix `STORE_ATTR_INSTANCE_VALUE` specialization check (GH-125612) The `STORE_ATTR_INSTANCE_VALUE` opcode doesn't support objects with non-NULL managed dictionaries, so don't specialize to that op in that case. --- Python/specialize.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/specialize.c b/Python/specialize.c index ec2cd7025e5054..d3fea717243847 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -947,7 +947,10 @@ specialize_dict_access( return 0; } _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && _PyObject_InlineValues(owner)->valid) { + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && + _PyObject_InlineValues(owner)->valid && + !(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL)) + { PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(PyUnicode_CheckExact(name)); Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); From 12680ec5bd45c85b6daebe0739d30ef45f089efa Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 6 Dec 2024 10:58:19 -0500 Subject: [PATCH 124/427] gh-127314: Don't mention the GIL when calling without a thread state on the free-threaded build (#127315) Co-authored-by: Victor Stinner --- Include/internal/pycore_pystate.h | 8 ++++++++ Lib/test/test_capi/test_mem.py | 9 +++++++-- Lib/test/test_capi/test_misc.py | 17 ++++++++++++----- ...24-11-26-22-06-10.gh-issue-127314.SsRrIu.rst | 2 ++ Objects/obmalloc.c | 7 +++++++ 5 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 54d8803bc0bdb6..1e73e541ef8de0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -190,10 +190,18 @@ static inline void _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) { if (tstate == NULL) { +#ifndef Py_GIL_DISABLED _Py_FatalErrorFunc(func, "the function must be called with the GIL held, " "after Python initialization and before Python finalization, " "but the GIL is released (the current Python thread state is NULL)"); +#else + _Py_FatalErrorFunc(func, + "the function must be called with an active thread state, " + "after Python initialization and before Python finalization, " + "but it was called without an active thread state. " + "Are you trying to call the C API inside of a Py_BEGIN_ALLOW_THREADS block?"); +#endif } } diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 6ab7b685c2e18b..5035b2b4829bf6 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -68,8 +68,13 @@ def test_api_misuse(self): def check_malloc_without_gil(self, code): out = self.check(code) - expected = ('Fatal Python error: _PyMem_DebugMalloc: ' - 'Python memory allocator called without holding the GIL') + if not support.Py_GIL_DISABLED: + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without holding the GIL') + else: + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without an active thread state. ' + 'Are you trying to call it inside of a Py_BEGIN_ALLOW_THREADS block?') self.assertIn(expected, out) def test_pymem_malloc_without_gil(self): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 8e0271919cc8a5..61512e610f46f2 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -100,11 +100,18 @@ def test_no_FatalError_infinite_loop(self): _rc, out, err = run_result self.assertEqual(out, b'') # This used to cause an infinite loop. - msg = ("Fatal Python error: PyThreadState_Get: " - "the function must be called with the GIL held, " - "after Python initialization and before Python finalization, " - "but the GIL is released " - "(the current Python thread state is NULL)").encode() + if not support.Py_GIL_DISABLED: + msg = ("Fatal Python error: PyThreadState_Get: " + "the function must be called with the GIL held, " + "after Python initialization and before Python finalization, " + "but the GIL is released " + "(the current Python thread state is NULL)").encode() + else: + msg = ("Fatal Python error: PyThreadState_Get: " + "the function must be called with an active thread state, " + "after Python initialization and before Python finalization, " + "but it was called without an active thread state. " + "Are you trying to call the C API inside of a Py_BEGIN_ALLOW_THREADS block?").encode() self.assertTrue(err.rstrip().startswith(msg), err) diff --git a/Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst b/Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst new file mode 100644 index 00000000000000..8ea3c4ee2a2c53 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst @@ -0,0 +1,2 @@ +Improve error message when calling the C API without an active thread state +on the :term:`free-threaded ` build. diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 2cc0377f68f990..b103deb01ca712 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2910,9 +2910,16 @@ static inline void _PyMem_DebugCheckGIL(const char *func) { if (!PyGILState_Check()) { +#ifndef Py_GIL_DISABLED _Py_FatalErrorFunc(func, "Python memory allocator called " "without holding the GIL"); +#else + _Py_FatalErrorFunc(func, + "Python memory allocator called " + "without an active thread state. " + "Are you trying to call it inside of a Py_BEGIN_ALLOW_THREADS block?"); +#endif } } From 67b18a18b66b89e253f38895057ef9f6bae92e7b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Dec 2024 17:27:12 +0100 Subject: [PATCH 125/427] gh-59705: Add _thread.set_name() function (#127338) On Linux, threading.Thread now sets the thread name to the operating system. * configure now checks if pthread_getname_np() and pthread_setname_np() functions are available. * Add PYTHREAD_NAME_MAXLEN macro. * Add _thread._NAME_MAXLEN constant for test_threading. Co-authored-by: Serhiy Storchaka --- Lib/test/test_threading.py | 60 ++++++++++ Lib/threading.py | 9 ++ ...4-11-27-17-04-38.gh-issue-59705.sAGyvs.rst | 2 + Modules/_threadmodule.c | 108 ++++++++++++++++++ Modules/clinic/_threadmodule.c.h | 104 +++++++++++++++++ configure | 30 +++++ configure.ac | 22 +++- pyconfig.h.in | 9 ++ 8 files changed, 342 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst create mode 100644 Modules/clinic/_threadmodule.c.h diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index fe225558fc4f0b..d05161f46f1034 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -2104,6 +2104,66 @@ def test__all__(self): support.check__all__(self, threading, ('threading', '_thread'), extra=extra, not_exported=not_exported) + @unittest.skipUnless(hasattr(_thread, 'set_name'), "missing _thread.set_name") + @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") + def test_set_name(self): + # set_name() limit in bytes + truncate = getattr(_thread, "_NAME_MAXLEN", None) + limit = truncate or 100 + + tests = [ + # test short ASCII name + "CustomName", + + # test short non-ASCII name + "namé€", + + # embedded null character: name is truncated + # at the first null character + "embed\0null", + + # Test long ASCII names (not truncated) + "x" * limit, + + # Test long ASCII names (truncated) + "x" * (limit + 10), + + # Test long non-ASCII name (truncated) + "x" * (limit - 1) + "é€", + ] + if os_helper.FS_NONASCII: + tests.append(f"nonascii:{os_helper.FS_NONASCII}") + if os_helper.TESTFN_UNENCODABLE: + tests.append(os_helper.TESTFN_UNENCODABLE) + + if sys.platform.startswith("solaris"): + encoding = "utf-8" + else: + encoding = sys.getfilesystemencoding() + + def work(): + nonlocal work_name + work_name = _thread._get_name() + + for name in tests: + encoded = name.encode(encoding, "replace") + if b'\0' in encoded: + encoded = encoded.split(b'\0', 1)[0] + if truncate is not None: + encoded = encoded[:truncate] + if sys.platform.startswith("solaris"): + expected = encoded.decode("utf-8", "surrogateescape") + else: + expected = os.fsdecode(encoded) + + with self.subTest(name=name, expected=expected): + work_name = None + thread = threading.Thread(target=work, name=name) + thread.start() + thread.join() + self.assertEqual(work_name, expected, + f"{len(work_name)=} and {len(expected)=}") + class InterruptMainTests(unittest.TestCase): def check_interrupt_main_with_signal_handler(self, signum): diff --git a/Lib/threading.py b/Lib/threading.py index 94ea2f08178369..3abd22a2aa1b72 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -48,6 +48,10 @@ __all__.append('get_native_id') except AttributeError: _HAVE_THREAD_NATIVE_ID = False +try: + _set_name = _thread.set_name +except AttributeError: + _set_name = None ThreadError = _thread.error try: _CRLock = _thread.RLock @@ -1027,6 +1031,11 @@ def _bootstrap_inner(self): self._set_ident() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() + if _set_name is not None and self._name: + try: + _set_name(self._name) + except OSError: + pass self._started.set() with _active_limbo_lock: _active[self._ident] = self diff --git a/Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst b/Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst new file mode 100644 index 00000000000000..a8c7b3d00755e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst @@ -0,0 +1,2 @@ +On Linux, :class:`threading.Thread` now sets the thread name to the +operating system. Patch by Victor Stinner. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 4a45445e2f62db..35c032fbeaa94f 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -17,6 +17,8 @@ # include // SIGINT #endif +#include "clinic/_threadmodule.c.h" + // ThreadError is just an alias to PyExc_RuntimeError #define ThreadError PyExc_RuntimeError @@ -44,6 +46,13 @@ get_thread_state(PyObject *module) return (thread_module_state *)state; } + +/*[clinic input] +module _thread +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=be8dbe5cc4b16df7]*/ + + // _ThreadHandle type // Handles state transitions according to the following diagram: @@ -2354,6 +2363,96 @@ PyDoc_STRVAR(thread__get_main_thread_ident_doc, Internal only. Return a non-zero integer that uniquely identifies the main thread\n\ of the main interpreter."); + +#ifdef HAVE_PTHREAD_GETNAME_NP +/*[clinic input] +_thread._get_name + +Get the name of the current thread. +[clinic start generated code]*/ + +static PyObject * +_thread__get_name_impl(PyObject *module) +/*[clinic end generated code: output=20026e7ee3da3dd7 input=35cec676833d04c8]*/ +{ + // Linux and macOS are limited to respectively 16 and 64 bytes + char name[100]; + pthread_t thread = pthread_self(); + int rc = pthread_getname_np(thread, name, Py_ARRAY_LENGTH(name)); + if (rc) { + errno = rc; + return PyErr_SetFromErrno(PyExc_OSError); + } + +#ifdef __sun + return PyUnicode_DecodeUTF8(name, strlen(name), "surrogateescape"); +#else + return PyUnicode_DecodeFSDefault(name); +#endif +} +#endif // HAVE_PTHREAD_GETNAME_NP + + +#ifdef HAVE_PTHREAD_SETNAME_NP +/*[clinic input] +_thread.set_name + + name as name_obj: unicode + +Set the name of the current thread. +[clinic start generated code]*/ + +static PyObject * +_thread_set_name_impl(PyObject *module, PyObject *name_obj) +/*[clinic end generated code: output=402b0c68e0c0daed input=7e7acd98261be82f]*/ +{ +#ifdef __sun + // Solaris always uses UTF-8 + const char *encoding = "utf-8"; +#else + // Encode the thread name to the filesystem encoding using the "replace" + // error handler + PyInterpreterState *interp = _PyInterpreterState_GET(); + const char *encoding = interp->unicode.fs_codec.encoding; +#endif + PyObject *name_encoded; + name_encoded = PyUnicode_AsEncodedString(name_obj, encoding, "replace"); + if (name_encoded == NULL) { + return NULL; + } + +#ifdef PYTHREAD_NAME_MAXLEN + // Truncate to PYTHREAD_NAME_MAXLEN bytes + the NUL byte if needed + size_t len = PyBytes_GET_SIZE(name_encoded); + if (len > PYTHREAD_NAME_MAXLEN) { + PyObject *truncated; + truncated = PyBytes_FromStringAndSize(PyBytes_AS_STRING(name_encoded), + PYTHREAD_NAME_MAXLEN); + if (truncated == NULL) { + Py_DECREF(name_encoded); + return NULL; + } + Py_SETREF(name_encoded, truncated); + } +#endif + + const char *name = PyBytes_AS_STRING(name_encoded); +#ifdef __APPLE__ + int rc = pthread_setname_np(name); +#else + pthread_t thread = pthread_self(); + int rc = pthread_setname_np(thread, name); +#endif + Py_DECREF(name_encoded); + if (rc) { + errno = rc; + return PyErr_SetFromErrno(PyExc_OSError); + } + Py_RETURN_NONE; +} +#endif // HAVE_PTHREAD_SETNAME_NP + + static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, METH_VARARGS, start_new_thread_doc}, @@ -2393,6 +2492,8 @@ static PyMethodDef thread_methods[] = { METH_O, thread__make_thread_handle_doc}, {"_get_main_thread_ident", thread__get_main_thread_ident, METH_NOARGS, thread__get_main_thread_ident_doc}, + _THREAD_SET_NAME_METHODDEF + _THREAD__GET_NAME_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -2484,6 +2585,13 @@ thread_module_exec(PyObject *module) llist_init(&state->shutdown_handles); +#ifdef PYTHREAD_NAME_MAXLEN + if (PyModule_AddIntConstant(module, "_NAME_MAXLEN", + PYTHREAD_NAME_MAXLEN) < 0) { + return -1; + } +#endif + return 0; } diff --git a/Modules/clinic/_threadmodule.c.h b/Modules/clinic/_threadmodule.c.h new file mode 100644 index 00000000000000..8f0507d40285b3 --- /dev/null +++ b/Modules/clinic/_threadmodule.c.h @@ -0,0 +1,104 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +#if defined(HAVE_PTHREAD_GETNAME_NP) + +PyDoc_STRVAR(_thread__get_name__doc__, +"_get_name($module, /)\n" +"--\n" +"\n" +"Get the name of the current thread."); + +#define _THREAD__GET_NAME_METHODDEF \ + {"_get_name", (PyCFunction)_thread__get_name, METH_NOARGS, _thread__get_name__doc__}, + +static PyObject * +_thread__get_name_impl(PyObject *module); + +static PyObject * +_thread__get_name(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _thread__get_name_impl(module); +} + +#endif /* defined(HAVE_PTHREAD_GETNAME_NP) */ + +#if defined(HAVE_PTHREAD_SETNAME_NP) + +PyDoc_STRVAR(_thread_set_name__doc__, +"set_name($module, /, name)\n" +"--\n" +"\n" +"Set the name of the current thread."); + +#define _THREAD_SET_NAME_METHODDEF \ + {"set_name", _PyCFunction_CAST(_thread_set_name), METH_FASTCALL|METH_KEYWORDS, _thread_set_name__doc__}, + +static PyObject * +_thread_set_name_impl(PyObject *module, PyObject *name_obj); + +static PyObject * +_thread_set_name(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_name", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *name_obj; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("set_name", "argument 'name'", "str", args[0]); + goto exit; + } + name_obj = args[0]; + return_value = _thread_set_name_impl(module, name_obj); + +exit: + return return_value; +} + +#endif /* defined(HAVE_PTHREAD_SETNAME_NP) */ + +#ifndef _THREAD__GET_NAME_METHODDEF + #define _THREAD__GET_NAME_METHODDEF +#endif /* !defined(_THREAD__GET_NAME_METHODDEF) */ + +#ifndef _THREAD_SET_NAME_METHODDEF + #define _THREAD_SET_NAME_METHODDEF +#endif /* !defined(_THREAD_SET_NAME_METHODDEF) */ +/*[clinic end generated code: output=b5cb85aaccc45bf6 input=a9049054013a1b77]*/ diff --git a/configure b/configure index 5e9bcb602d884e..bcbab8dfcff190 100755 --- a/configure +++ b/configure @@ -821,6 +821,7 @@ MODULE_TIME_TRUE MODULE__IO_FALSE MODULE__IO_TRUE MODULE_BUILDTYPE +PYTHREAD_NAME_MAXLEN TEST_MODULES OPENSSL_LDFLAGS OPENSSL_LIBS @@ -18841,6 +18842,18 @@ if test "x$ac_cv_func_pthread_kill" = xyes then : printf "%s\n" "#define HAVE_PTHREAD_KILL 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "pthread_getname_np" "ac_cv_func_pthread_getname_np" +if test "x$ac_cv_func_pthread_getname_np" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_GETNAME_NP 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "pthread_setname_np" "ac_cv_func_pthread_setname_np" +if test "x$ac_cv_func_pthread_setname_np" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_SETNAME_NP 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "ptsname" "ac_cv_func_ptsname" if test "x$ac_cv_func_ptsname" = xyes @@ -29081,6 +29094,23 @@ fi CPPFLAGS=$save_CPPFLAGS +# gh-59705: Maximum length in bytes of a thread name +case "$ac_sys_system" in + Linux*) PYTHREAD_NAME_MAXLEN=15;; # Linux and Android + SunOS*) PYTHREAD_NAME_MAXLEN=31;; + Darwin) PYTHREAD_NAME_MAXLEN=63;; + iOS) PYTHREAD_NAME_MAXLEN=63;; + FreeBSD*) PYTHREAD_NAME_MAXLEN=98;; + *) PYTHREAD_NAME_MAXLEN=;; +esac +if test -n "$PYTHREAD_NAME_MAXLEN"; then + +printf "%s\n" "#define PYTHREAD_NAME_MAXLEN $PYTHREAD_NAME_MAXLEN" >>confdefs.h + +fi + + + # stdlib diff --git a/configure.ac b/configure.ac index bf3685e1b1b209..922a125ea9608e 100644 --- a/configure.ac +++ b/configure.ac @@ -5110,8 +5110,10 @@ AC_CHECK_FUNCS([ \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ - pread preadv preadv2 process_vm_readv pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ - pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + pread preadv preadv2 process_vm_readv \ + pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ + pthread_kill pthread_getname_np pthread_setname_np \ + ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \ @@ -7498,6 +7500,22 @@ AS_VAR_IF([ac_cv_libatomic_needed], [yes], _RESTORE_VAR([CPPFLAGS]) +# gh-59705: Maximum length in bytes of a thread name +case "$ac_sys_system" in + Linux*) PYTHREAD_NAME_MAXLEN=15;; # Linux and Android + SunOS*) PYTHREAD_NAME_MAXLEN=31;; + Darwin) PYTHREAD_NAME_MAXLEN=63;; + iOS) PYTHREAD_NAME_MAXLEN=63;; + FreeBSD*) PYTHREAD_NAME_MAXLEN=98;; + *) PYTHREAD_NAME_MAXLEN=;; +esac +if test -n "$PYTHREAD_NAME_MAXLEN"; then + AC_DEFINE_UNQUOTED([PYTHREAD_NAME_MAXLEN], [$PYTHREAD_NAME_MAXLEN], + [Maximum length in bytes of a thread name]) +fi +AC_SUBST([PYTHREAD_NAME_MAXLEN]) + + # stdlib AC_DEFUN([PY_STDLIB_MOD_SET_NA], [ m4_foreach([mod], [$@], [ diff --git a/pyconfig.h.in b/pyconfig.h.in index 6a1f1284650b9f..166c195a8c66fc 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -981,6 +981,9 @@ /* Define to 1 if you have the `pthread_getcpuclockid' function. */ #undef HAVE_PTHREAD_GETCPUCLOCKID +/* Define to 1 if you have the `pthread_getname_np' function. */ +#undef HAVE_PTHREAD_GETNAME_NP + /* Define to 1 if you have the header file. */ #undef HAVE_PTHREAD_H @@ -990,6 +993,9 @@ /* Define to 1 if you have the `pthread_kill' function. */ #undef HAVE_PTHREAD_KILL +/* Define to 1 if you have the `pthread_setname_np' function. */ +#undef HAVE_PTHREAD_SETNAME_NP + /* Define to 1 if you have the `pthread_sigmask' function. */ #undef HAVE_PTHREAD_SIGMASK @@ -1650,6 +1656,9 @@ /* Define as the preferred size in bits of long digits */ #undef PYLONG_BITS_IN_DIGIT +/* Maximum length in bytes of a thread name */ +#undef PYTHREAD_NAME_MAXLEN + /* enabled builtin hash modules */ #undef PY_BUILTIN_HASHLIB_HASHES From 89fa7ec74e531870a8f495d5e32ec0b00dbcd32b Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:36:06 +0000 Subject: [PATCH 126/427] gh-119786: Add jit.md. Move adaptive.md to a section of interpreter.md. (#127175) --- InternalDocs/README.md | 4 +- InternalDocs/adaptive.md | 146 ------------------------ InternalDocs/code_objects.md | 5 + InternalDocs/compiler.md | 10 -- InternalDocs/interpreter.md | 210 ++++++++++++++++++++++++++++++----- InternalDocs/jit.md | 134 ++++++++++++++++++++++ 6 files changed, 322 insertions(+), 187 deletions(-) delete mode 100644 InternalDocs/adaptive.md create mode 100644 InternalDocs/jit.md diff --git a/InternalDocs/README.md b/InternalDocs/README.md index 8cdd06d189f362..794b4f3c6aad42 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -34,9 +34,9 @@ Runtime Objects Program Execution --- -- [The Interpreter](interpreter.md) +- [The Bytecode Interpreter](interpreter.md) -- [Adaptive Instruction Families](adaptive.md) +- [The JIT](jit.md) - [Garbage Collector Design](garbage_collector.md) diff --git a/InternalDocs/adaptive.md b/InternalDocs/adaptive.md deleted file mode 100644 index 7cfa8e52310460..00000000000000 --- a/InternalDocs/adaptive.md +++ /dev/null @@ -1,146 +0,0 @@ -# Adding or extending a family of adaptive instructions. - -## Families of instructions - -The core part of [PEP 659](https://peps.python.org/pep-0659/) -(specializing adaptive interpreter) is the families of -instructions that perform the adaptive specialization. - -A family of instructions has the following fundamental properties: - -* It corresponds to a single instruction in the code - generated by the bytecode compiler. -* It has a single adaptive instruction that records an execution count and, - at regular intervals, attempts to specialize itself. If not specializing, - it executes the base implementation. -* It has at least one specialized form of the instruction that is tailored - for a particular value or set of values at runtime. -* All members of the family must have the same number of inline cache entries, - to ensure correct execution. - Individual family members do not need to use all of the entries, - but must skip over any unused entries when executing. - -The current implementation also requires the following, -although these are not fundamental and may change: - -* All families use one or more inline cache entries, - the first entry is always the counter. -* All instruction names should start with the name of the adaptive - instruction. -* Specialized forms should have names describing their specialization. - -## Example family - -The `LOAD_GLOBAL` instruction (in [Python/bytecodes.c](../Python/bytecodes.c)) -already has an adaptive family that serves as a relatively simple example. - -The `LOAD_GLOBAL` instruction performs adaptive specialization, -calling `_Py_Specialize_LoadGlobal()` when the counter reaches zero. - -There are two specialized instructions in the family, `LOAD_GLOBAL_MODULE` -which is specialized for global variables in the module, and -`LOAD_GLOBAL_BUILTIN` which is specialized for builtin variables. - -## Performance analysis - -The benefit of a specialization can be assessed with the following formula: -`Tbase/Tadaptive`. - -Where `Tbase` is the mean time to execute the base instruction, -and `Tadaptive` is the mean time to execute the specialized and adaptive forms. - -`Tadaptive = (sum(Ti*Ni) + Tmiss*Nmiss)/(sum(Ni)+Nmiss)` - -`Ti` is the time to execute the `i`th instruction in the family and `Ni` is -the number of times that instruction is executed. -`Tmiss` is the time to process a miss, including de-optimzation -and the time to execute the base instruction. - -The ideal situation is where misses are rare and the specialized -forms are much faster than the base instruction. -`LOAD_GLOBAL` is near ideal, `Nmiss/sum(Ni) ≈ 0`. -In which case we have `Tadaptive ≈ sum(Ti*Ni)`. -Since we can expect the specialized forms `LOAD_GLOBAL_MODULE` and -`LOAD_GLOBAL_BUILTIN` to be much faster than the adaptive base instruction, -we would expect the specialization of `LOAD_GLOBAL` to be profitable. - -## Design considerations - -While `LOAD_GLOBAL` may be ideal, instructions like `LOAD_ATTR` and -`CALL_FUNCTION` are not. For maximum performance we want to keep `Ti` -low for all specialized instructions and `Nmiss` as low as possible. - -Keeping `Nmiss` low means that there should be specializations for almost -all values seen by the base instruction. Keeping `sum(Ti*Ni)` low means -keeping `Ti` low which means minimizing branches and dependent memory -accesses (pointer chasing). These two objectives may be in conflict, -requiring judgement and experimentation to design the family of instructions. - -The size of the inline cache should as small as possible, -without impairing performance, to reduce the number of -`EXTENDED_ARG` jumps, and to reduce pressure on the CPU's data cache. - -### Gathering data - -Before choosing how to specialize an instruction, it is important to gather -some data. What are the patterns of usage of the base instruction? -Data can best be gathered by instrumenting the interpreter. Since a -specialization function and adaptive instruction are going to be required, -instrumentation can most easily be added in the specialization function. - -### Choice of specializations - -The performance of the specializing adaptive interpreter relies on the -quality of specialization and keeping the overhead of specialization low. - -Specialized instructions must be fast. In order to be fast, -specialized instructions should be tailored for a particular -set of values that allows them to: - -1. Verify that incoming value is part of that set with low overhead. -2. Perform the operation quickly. - -This requires that the set of values is chosen such that membership can be -tested quickly and that membership is sufficient to allow the operation to -performed quickly. - -For example, `LOAD_GLOBAL_MODULE` is specialized for `globals()` -dictionaries that have a keys with the expected version. - -This can be tested quickly: - -* `globals->keys->dk_version == expected_version` - -and the operation can be performed quickly: - -* `value = entries[cache->index].me_value;`. - -Because it is impossible to measure the performance of an instruction without -also measuring unrelated factors, the assessment of the quality of a -specialization will require some judgement. - -As a general rule, specialized instructions should be much faster than the -base instruction. - -### Implementation of specialized instructions - -In general, specialized instructions should be implemented in two parts: - -1. A sequence of guards, each of the form - `DEOPT_IF(guard-condition-is-false, BASE_NAME)`. -2. The operation, which should ideally have no branches and - a minimum number of dependent memory accesses. - -In practice, the parts may overlap, as data required for guards -can be re-used in the operation. - -If there are branches in the operation, then consider further specialization -to eliminate the branches. - -### Maintaining stats - -Finally, take care that stats are gather correctly. -After the last `DEOPT_IF` has passed, a hit should be recorded with -`STAT_INC(BASE_INSTRUCTION, hit)`. -After an optimization has been deferred in the adaptive instruction, -that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. diff --git a/InternalDocs/code_objects.md b/InternalDocs/code_objects.md index d4e28c6b238b48..a91a7043c1b8d4 100644 --- a/InternalDocs/code_objects.md +++ b/InternalDocs/code_objects.md @@ -18,6 +18,11 @@ Code objects are typically produced by the bytecode [compiler](compiler.md), although they are often written to disk by one process and read back in by another. The disk version of a code object is serialized using the [marshal](https://docs.python.org/dev/library/marshal.html) protocol. +When a `CodeObject` is created, the function `_PyCode_Quicken()` from +[`Python/specialize.c`](../Python/specialize.c) is called to initialize +the caches of all adaptive instructions. This is required because the +on-disk format is a sequence of bytes, and some of the caches need to be +initialized with 16-bit values. Code objects are nominally immutable. Some fields (including `co_code_adaptive` and fields for runtime diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 9e99f348acbd8f..c257bfd9faf78f 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -595,16 +595,6 @@ Objects * [Exception Handling](exception_handling.md): Describes the exception table -Specializing Adaptive Interpreter -================================= - -Adding a specializing, adaptive interpreter to CPython will bring significant -performance improvements. These documents provide more information: - -* [PEP 659: Specializing Adaptive Interpreter](https://peps.python.org/pep-0659/). -* [Adding or extending a family of adaptive instructions](adaptive.md) - - References ========== diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index ab149e43471072..fa4a54fdc54fac 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -1,8 +1,4 @@ -The bytecode interpreter -======================== - -Overview --------- +# The bytecode interpreter This document describes the workings and implementation of the bytecode interpreter, the part of python that executes compiled Python code. Its @@ -47,8 +43,7 @@ simply calls [`_PyEval_EvalFrameDefault()`] to execute the frame. However, as pe `_PyEval_EvalFrameDefault()`. -Instruction decoding --------------------- +## Instruction decoding The first task of the interpreter is to decode the bytecode instructions. Bytecode is stored as an array of 16-bit code units (`_Py_CODEUNIT`). @@ -110,8 +105,7 @@ snippet decode a complete instruction: For various reasons we'll get to later (mostly efficiency, given that `EXTENDED_ARG` is rare) the actual code is different. -Jumps -===== +## Jumps Note that when the `switch` statement is reached, `next_instr` (the "instruction offset") already points to the next instruction. @@ -120,25 +114,26 @@ Thus, jump instructions can be implemented by manipulating `next_instr`: - A jump forward (`JUMP_FORWARD`) sets `next_instr += oparg`. - A jump backward sets `next_instr -= oparg`. -Inline cache entries -==================== +## Inline cache entries Some (specialized or specializable) instructions have an associated "inline cache". The inline cache consists of one or more two-byte entries included in the bytecode array as additional words following the `opcode`/`oparg` pair. The size of the inline cache for a particular instruction is fixed by its `opcode`. Moreover, the inline cache size for all instructions in a -[family of specialized/specializable instructions](adaptive.md) +[family of specialized/specializable instructions](#Specialization) (for example, `LOAD_ATTR`, `LOAD_ATTR_SLOT`, `LOAD_ATTR_MODULE`) must all be the same. Cache entries are reserved by the compiler and initialized with zeros. Although they are represented by code units, cache entries do not conform to the `opcode` / `oparg` format. -If an instruction has an inline cache, the layout of its cache is described by -a `struct` definition in (`pycore_code.h`)[../Include/internal/pycore_code.h]. -This allows us to access the cache by casting `next_instr` to a pointer to this `struct`. -The size of such a `struct` must be independent of the machine architecture, word size -and alignment requirements. For a 32-bit field, the `struct` should use `_Py_CODEUNIT field[2]`. +If an instruction has an inline cache, the layout of its cache is described in +the instruction's definition in [`Python/bytecodes.c`](../Python/bytecodes.c). +The structs defined in [`pycore_code.h`](../Include/internal/pycore_code.h) +allow us to access the cache by casting `next_instr` to a pointer to the relevant +`struct`. The size of such a `struct` must be independent of the machine +architecture, word size and alignment requirements. For a 32-bit field, the +`struct` should use `_Py_CODEUNIT field[2]`. The instruction implementation is responsible for advancing `next_instr` past the inline cache. For example, if an instruction's inline cache is four bytes (that is, two code units) in size, @@ -153,8 +148,7 @@ Serializing non-zero cache entries would present a problem because the serializa More information about the use of inline caches can be found in [PEP 659](https://peps.python.org/pep-0659/#ancillary-data). -The evaluation stack --------------------- +## The evaluation stack Most instructions read or write some data in the form of object references (`PyObject *`). The CPython bytecode interpreter is a stack machine, meaning that its instructions operate @@ -193,16 +187,14 @@ For example, the following sequence is illegal, because it keeps pushing items o > Do not confuse the evaluation stack with the call stack, which is used to implement calling > and returning from functions. -Error handling --------------- +## Error handling When the implementation of an opcode raises an exception, it jumps to the `exception_unwind` label in [Python/ceval.c](../Python/ceval.c). The exception is then handled as described in the [`exception handling documentation`](exception_handling.md#handling-exceptions). -Python-to-Python calls ----------------------- +## Python-to-Python calls The `_PyEval_EvalFrameDefault()` function is recursive, because sometimes the interpreter calls some C function that calls back into the interpreter. @@ -227,8 +219,7 @@ returns from `_PyEval_EvalFrameDefault()` altogether, to a C caller. A similar check is performed when an unhandled exception occurs. -The call stack --------------- +## The call stack Up through 3.10, the call stack was implemented as a singly-linked list of [frame objects](frames.md). This was expensive because each call would require a @@ -262,8 +253,7 @@ See also the [generators](generators.md) section. -Introducing a new bytecode instruction --------------------------------------- +## Introducing a new bytecode instruction It is occasionally necessary to add a new opcode in order to implement a new feature or change the way that existing features are compiled. @@ -355,6 +344,169 @@ new bytecode properly. Run `make regen-importlib` for updating the bytecode of frozen importlib files. You have to run `make` again after this to recompile the generated C files. +## Specialization + +Bytecode specialization, which was introduced in +[PEP 659](https://peps.python.org/pep-0659/), speeds up program execution by +rewriting instructions based on runtime information. This is done by replacing +a generic instruction with a faster version that works for the case that this +program encounters. Each specializable instruction is responsible for rewriting +itself, using its [inline caches](#inline-cache-entries) for +bookkeeping. + +When an adaptive instruction executes, it may attempt to specialize itself, +depending on the argument and the contents of its cache. This is done +by calling one of the `_Py_Specialize_XXX` functions in +[`Python/specialize.c`](../Python/specialize.c). + + +The specialized instructions are responsible for checking that the special-case +assumptions still apply, and de-optimizing back to the generic version if not. + +## Families of instructions + +A *family* of instructions consists of an adaptive instruction along with the +specialized instructions that it can be replaced by. +It has the following fundamental properties: + +* It corresponds to a single instruction in the code + generated by the bytecode compiler. +* It has a single adaptive instruction that records an execution count and, + at regular intervals, attempts to specialize itself. If not specializing, + it executes the base implementation. +* It has at least one specialized form of the instruction that is tailored + for a particular value or set of values at runtime. +* All members of the family must have the same number of inline cache entries, + to ensure correct execution. + Individual family members do not need to use all of the entries, + but must skip over any unused entries when executing. + +The current implementation also requires the following, +although these are not fundamental and may change: + +* All families use one or more inline cache entries, + the first entry is always the counter. +* All instruction names should start with the name of the adaptive + instruction. +* Specialized forms should have names describing their specialization. + +## Example family + +The `LOAD_GLOBAL` instruction (in [Python/bytecodes.c](../Python/bytecodes.c)) +already has an adaptive family that serves as a relatively simple example. + +The `LOAD_GLOBAL` instruction performs adaptive specialization, +calling `_Py_Specialize_LoadGlobal()` when the counter reaches zero. + +There are two specialized instructions in the family, `LOAD_GLOBAL_MODULE` +which is specialized for global variables in the module, and +`LOAD_GLOBAL_BUILTIN` which is specialized for builtin variables. + +## Performance analysis + +The benefit of a specialization can be assessed with the following formula: +`Tbase/Tadaptive`. + +Where `Tbase` is the mean time to execute the base instruction, +and `Tadaptive` is the mean time to execute the specialized and adaptive forms. + +`Tadaptive = (sum(Ti*Ni) + Tmiss*Nmiss)/(sum(Ni)+Nmiss)` + +`Ti` is the time to execute the `i`th instruction in the family and `Ni` is +the number of times that instruction is executed. +`Tmiss` is the time to process a miss, including de-optimzation +and the time to execute the base instruction. + +The ideal situation is where misses are rare and the specialized +forms are much faster than the base instruction. +`LOAD_GLOBAL` is near ideal, `Nmiss/sum(Ni) ≈ 0`. +In which case we have `Tadaptive ≈ sum(Ti*Ni)`. +Since we can expect the specialized forms `LOAD_GLOBAL_MODULE` and +`LOAD_GLOBAL_BUILTIN` to be much faster than the adaptive base instruction, +we would expect the specialization of `LOAD_GLOBAL` to be profitable. + +## Design considerations + +While `LOAD_GLOBAL` may be ideal, instructions like `LOAD_ATTR` and +`CALL_FUNCTION` are not. For maximum performance we want to keep `Ti` +low for all specialized instructions and `Nmiss` as low as possible. + +Keeping `Nmiss` low means that there should be specializations for almost +all values seen by the base instruction. Keeping `sum(Ti*Ni)` low means +keeping `Ti` low which means minimizing branches and dependent memory +accesses (pointer chasing). These two objectives may be in conflict, +requiring judgement and experimentation to design the family of instructions. + +The size of the inline cache should as small as possible, +without impairing performance, to reduce the number of +`EXTENDED_ARG` jumps, and to reduce pressure on the CPU's data cache. + +### Gathering data + +Before choosing how to specialize an instruction, it is important to gather +some data. What are the patterns of usage of the base instruction? +Data can best be gathered by instrumenting the interpreter. Since a +specialization function and adaptive instruction are going to be required, +instrumentation can most easily be added in the specialization function. + +### Choice of specializations + +The performance of the specializing adaptive interpreter relies on the +quality of specialization and keeping the overhead of specialization low. + +Specialized instructions must be fast. In order to be fast, +specialized instructions should be tailored for a particular +set of values that allows them to: + +1. Verify that incoming value is part of that set with low overhead. +2. Perform the operation quickly. + +This requires that the set of values is chosen such that membership can be +tested quickly and that membership is sufficient to allow the operation to +performed quickly. + +For example, `LOAD_GLOBAL_MODULE` is specialized for `globals()` +dictionaries that have a keys with the expected version. + +This can be tested quickly: + +* `globals->keys->dk_version == expected_version` + +and the operation can be performed quickly: + +* `value = entries[cache->index].me_value;`. + +Because it is impossible to measure the performance of an instruction without +also measuring unrelated factors, the assessment of the quality of a +specialization will require some judgement. + +As a general rule, specialized instructions should be much faster than the +base instruction. + +### Implementation of specialized instructions + +In general, specialized instructions should be implemented in two parts: + +1. A sequence of guards, each of the form + `DEOPT_IF(guard-condition-is-false, BASE_NAME)`. +2. The operation, which should ideally have no branches and + a minimum number of dependent memory accesses. + +In practice, the parts may overlap, as data required for guards +can be re-used in the operation. + +If there are branches in the operation, then consider further specialization +to eliminate the branches. + +### Maintaining stats + +Finally, take care that stats are gathered correctly. +After the last `DEOPT_IF` has passed, a hit should be recorded with +`STAT_INC(BASE_INSTRUCTION, hit)`. +After an optimization has been deferred in the adaptive instruction, +that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. + + Additional resources -------------------- diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md new file mode 100644 index 00000000000000..1e9f385d5f87fa --- /dev/null +++ b/InternalDocs/jit.md @@ -0,0 +1,134 @@ +# The JIT + +The [adaptive interpreter](interpreter.md) consists of a main loop that +executes the bytecode instructions generated by the +[bytecode compiler](compiler.md) and their +[specializations](interpreter.md#Specialization). Runtime optimization in +this interpreter can only be done for one instruction at a time. The JIT +is based on a mechanism to replace an entire sequence of bytecode instructions, +and this enables optimizations that span multiple instructions. + +Historically, the adaptive interpreter was referred to as `tier 1` and +the JIT as `tier 2`. You will see remnants of this in the code. + +## The Optimizer and Executors + +The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` +instruction determines that it is "hot" because the counter in its +[inline cache](interpreter.md#inline-cache-entries) indicates that it +executed more than some threshold number of times (see +[`backoff_counter_triggers`](../Include/internal/pycore_backoff.h)). +It then calls the function `_PyOptimizer_Optimize()` in +[`Python/optimizer.c`](../Python/optimizer.c), passing it the current +[frame](frames.md) and instruction pointer. `_PyOptimizer_Optimize()` +constructs an object of type +[`_PyExecutorObject`](Include/internal/pycore_optimizer.h) which implements +an optimized version of the instruction trace beginning at this jump. + +The optimizer determines where the trace ends, and the executor is set up +to either return to the adaptive interpreter and resume execution, or +transfer control to another executor (see `_PyExitData` in +Include/internal/pycore_optimizer.h). + +The executor is stored on the [`code object`](code_objects.md) of the frame, +in the `co_executors` field which is an array of executors. The start +instruction of the trace (the `JUMP_BACKWARD`) is replaced by an +`ENTER_EXECUTOR` instruction whose `oparg` is equal to the index of the +executor in `co_executors`. + +## The micro-op optimizer + +The optimizer that `_PyOptimizer_Optimize()` runs is configurable via the +`_Py_SetTier2Optimizer()` function (this is used in test via +`_testinternalcapi.set_optimizer()`.) + +The micro-op (abbreviated `uop` to approximate `μop`) optimizer is defined in +[`Python/optimizer.c`](../Python/optimizer.c) as the type `_PyUOpOptimizer_Type`. +It translates an instruction trace into a sequence of micro-ops by replacing +each bytecode by an equivalent sequence of micro-ops (see +`_PyOpcode_macro_expansion` in +[pycore_opcode_metadata.h](../Include/internal/pycore_opcode_metadata.h) +which is generated from [`Python/bytecodes.c`](../Python/bytecodes.c)). +The micro-op sequence is then optimized by +`_Py_uop_analyze_and_optimize` in +[`Python/optimizer_analysis.c`](../Python/optimizer_analysis.c) +and an instance of `_PyUOpExecutor_Type` is created to contain it. + +## The JIT interpreter + +After a `JUMP_BACKWARD` instruction invokes the uop optimizer to create a uop +executor, it transfers control to this executor via the `GOTO_TIER_TWO` macro. + +CPython implements two executors. Here we describe the JIT interpreter, +which is the simpler of them and is therefore useful for debugging and analyzing +the uops generation and optimization stages. To run it, we configure the +JIT to run on its interpreter (i.e., python is configured with +[`--enable-experimental-jit=interpreter`](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit)). + +When invoked, the executor jumps to the `tier2_dispatch:` label in +[`Python/ceval.c`](../Python/ceval.c), where there is a loop that +executes the micro-ops. The body of this loop is a switch statement over +the uops IDs, resembling the one used in the adaptive interpreter. + +The swtich implementing the uops is in [`Python/executor_cases.c.h`](../Python/executor_cases.c.h), +which is generated by the build script +[`Tools/cases_generator/tier2_generator.py`](../Tools/cases_generator/tier2_generator.py) +from the bytecode definitions in +[`Python/bytecodes.c`](../Python/bytecodes.c). + +When an `_EXIT_TRACE` or `_DEOPT` uop is reached, the uop interpreter exits +and execution returns to the adaptive interpreter. + +## Invalidating Executors + +In addition to being stored on the code object, each executor is also +inserted into a list of all executors, which is stored in the interpreter +state's `executor_list_head` field. This list is used when it is necessary +to invalidate executors because values they used in their construction may +have changed. + +## The JIT + +When the full jit is enabled (python was configured with +[`--enable-experimental-jit`](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit), +the uop executor's `jit_code` field is populated with a pointer to a compiled +C function that implements the executor logic. This function's signature is +defined by `jit_func` in [`pycore_jit.h`](Include/internal/pycore_jit.h). +When the executor is invoked by `ENTER_EXECUTOR`, instead of jumping to +the uop interpreter at `tier2_dispatch`, the executor runs the function +that `jit_code` points to. This function returns the instruction pointer +of the next Tier 1 instruction that needs to execute. + +The generation of the jitted functions uses the copy-and-patch technique +which is described in +[Haoran Xu's article](https://sillycross.github.io/2023/05/12/2023-05-12/). +At its core are statically generated `stencils` for the implementation +of the micro ops, which are completed with runtime information while +the jitted code is constructed for an executor by +[`_PyJIT_Compile`](../Python/jit.c). + +The stencils are generated at build time under the Makefile target `regen-jit` +by the scripts in [`/Tools/jit`](/Tools/jit). This script reads +[`Python/executor_cases.c.h`](../Python/executor_cases.c.h) (which is +generated from [`Python/bytecodes.c`](../Python/bytecodes.c)). For +each opcode, it constructs a `.c` file that contains a function for +implementing this opcode, with some runtime information injected. +This is done by replacing `CASE` by the bytecode definition in the +template file [`Tools/jit/template.c`](../Tools/jit/template.c). + +Each of the `.c` files is compiled by LLVM, to produce an object file +that contains a function that executes the opcode. These compiled +functions are used to generate the file +[`jit_stencils.h`](../jit_stencils.h), which contains the functions +that the JIT can use to emit code for each of the bytecodes. + +For Python maintainers this means that changes to the bytecodes and +their implementations do not require changes related to the stencils, +because everything is automatically generated from +[`Python/bytecodes.c`](../Python/bytecodes.c) at build time. + +See Also: + +* [Copy-and-Patch Compilation: A fast compilation algorithm for high-level languages and bytecode](https://arxiv.org/abs/2011.13127) + +* [PyCon 2024: Building a JIT compiler for CPython](https://www.youtube.com/watch?v=kMO3Ju0QCDo) From e59caf67cdb8dae26470f00599ea8dbb00968a73 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:50:58 +0000 Subject: [PATCH 127/427] Fix typo in `Lib/_android_support.py` (#127699) --- Lib/_android_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 7572745c851847..ae506f6a4b57b8 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -6,7 +6,7 @@ # The maximum length of a log message in bytes, including the level marker and # tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71. -# Messages longer than this will be be truncated by logcat. This limit has already +# Messages longer than this will be truncated by logcat. This limit has already # been reduced at least once in the history of Android (from 4076 to 4068 between # API level 23 and 26), so leave some headroom. MAX_BYTES_PER_WRITE = 4000 From 5b6635f772d187d6049a56bfea76855644cd4ca1 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 6 Dec 2024 18:10:00 +0000 Subject: [PATCH 128/427] GH-127381: pathlib ABCs: remove `PathBase.rename()` and `replace()` (#127658) These methods are obviated by `PathBase.move()`, which can move directories and supports any `PathBase` object as a target. --- Lib/pathlib/_abc.py | 37 +---------------------- Lib/pathlib/_local.py | 17 +++++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 2 -- 3 files changed, 18 insertions(+), 38 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 86617ff2616f33..11a11ecc4c8203 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -14,7 +14,7 @@ import functools import operator import posixpath -from errno import EINVAL, EXDEV +from errno import EINVAL from glob import _GlobberBase, _no_recurse_symlinks from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from pathlib._os import copyfileobj @@ -902,45 +902,10 @@ def copy_into(self, target_dir, *, follow_symlinks=True, dirs_exist_ok=dirs_exist_ok, preserve_metadata=preserve_metadata) - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - raise UnsupportedOperation(self._unsupported_msg('rename()')) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - raise UnsupportedOperation(self._unsupported_msg('replace()')) - def move(self, target): """ Recursively move this file or directory tree to the given destination. """ - self._ensure_different_file(target) - try: - return self.replace(target) - except UnsupportedOperation: - pass - except TypeError: - if not isinstance(target, PathBase): - raise - except OSError as err: - if err.errno != EXDEV: - raise target = self.copy(target, follow_symlinks=False, preserve_metadata=True) self._delete() return target diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index bb8a252c0e94e2..250bc12956f5bc 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -4,6 +4,7 @@ import os import posixpath import sys +from errno import EXDEV from glob import _StringGlobber from itertools import chain from _collections_abc import Sequence @@ -876,6 +877,22 @@ def replace(self, target): os.replace(self, target) return self.with_segments(target) + def move(self, target): + """ + Recursively move this file or directory tree to the given destination. + """ + self._ensure_different_file(target) + try: + return self.replace(target) + except TypeError: + if not isinstance(target, PathBase): + raise + except OSError as err: + if err.errno != EXDEV: + raise + # Fall back to copy+delete. + return PathBase.move(self, target) + if hasattr(os, "symlink"): def symlink_to(self, target, target_is_directory=False): """ diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 7ba3fa823a30b9..00153e3f5e997e 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1376,8 +1376,6 @@ def test_unsupported_operation(self): self.assertRaises(e, p.hardlink_to, 'foo') self.assertRaises(e, p.mkdir) self.assertRaises(e, p.touch) - self.assertRaises(e, p.rename, 'foo') - self.assertRaises(e, p.replace, 'foo') self.assertRaises(e, p.chmod, 0o755) self.assertRaises(e, p.lchmod, 0o755) self.assertRaises(e, p.unlink) From 0fc4063747c96223575f6f5a0562eddf2ed0ed62 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 6 Dec 2024 10:42:05 -0800 Subject: [PATCH 129/427] GH-127652: stop using `--wasi preview2` in `wasi.py` (GH-127704) It's only to use WASI 0.2 code to back preview1 APIs and is considered experimental anyway. --- Tools/wasm/wasi.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index ac36d55587a38f..da847c4ff86215 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -297,8 +297,6 @@ def main(): # build. # Use 16 MiB stack. "--wasm max-wasm-stack=16777216 " - # Use WASI 0.2 primitives. - "--wasi preview2 " # Enable thread support; causes use of preview1. #"--wasm threads=y --wasi threads=y " # Map the checkout to / to load the stdlib from /Lib. From 31c9f3ced293492b38e784c17c4befe425da5dab Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 6 Dec 2024 21:39:45 +0000 Subject: [PATCH 130/427] GH-127381: pathlib ABCs: remove `PathBase.resolve()` and `absolute()` (#127707) Remove our implementation of POSIX path resolution in `PathBase.resolve()`. This functionality is rather fragile and isn't necessary in most cases. It depends on `PathBase.stat()`, which we're looking to remove. Also remove `PathBase.absolute()`. Many legitimate virtual filesystems lack the notion of a 'current directory', so it's wrong to include in the basic interface. --- Lib/pathlib/_abc.py | 64 +- Lib/test/test_pathlib/test_pathlib.py | 586 ++++++++++++++++++- Lib/test/test_pathlib/test_pathlib_abc.py | 680 +--------------------- 3 files changed, 599 insertions(+), 731 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 11a11ecc4c8203..820970fcd5889b 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -13,7 +13,6 @@ import functools import operator -import posixpath from errno import EINVAL from glob import _GlobberBase, _no_recurse_symlinks from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO @@ -115,11 +114,6 @@ class PurePathBase: # The `_raw_paths` slot stores unjoined string paths. This is set in # the `__init__()` method. '_raw_paths', - - # The '_resolving' slot stores a boolean indicating whether the path - # is being processed by `PathBase.resolve()`. This prevents duplicate - # work from occurring when `resolve()` calls `stat()` or `readlink()`. - '_resolving', ) parser = ParserBase() _globber = PathGlobber @@ -130,7 +124,6 @@ def __init__(self, *args): raise TypeError( f"argument should be a str, not {type(arg).__name__!r}") self._raw_paths = list(args) - self._resolving = False def with_segments(self, *pathsegments): """Construct a new path object from any number of path-like objects. @@ -339,9 +332,7 @@ def parent(self): path = str(self) parent = self.parser.split(path)[0] if path != parent: - parent = self.with_segments(parent) - parent._resolving = self._resolving - return parent + return self.with_segments(parent) return self @property @@ -424,9 +415,6 @@ class PathBase(PurePathBase): """ __slots__ = () - # Maximum number of symlinks to follow in resolve() - _max_symlinks = 40 - @classmethod def _unsupported_msg(cls, attribute): return f"{cls.__name__}.{attribute} is unsupported" @@ -720,20 +708,6 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): yield path, dirnames, filenames paths += [path.joinpath(d) for d in reversed(dirnames)] - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - if self.is_absolute(): - return self - elif self.parser is not posixpath: - raise UnsupportedOperation(self._unsupported_msg('absolute()')) - else: - # Treat the root directory as the current working directory. - return self.with_segments('/', *self._raw_paths) - def expanduser(self): """ Return a new path with expanded ~ and ~user constructs (as returned by os.path.expanduser) @@ -745,42 +719,6 @@ def readlink(self): Return the path to which the symbolic link points. """ raise UnsupportedOperation(self._unsupported_msg('readlink()')) - readlink._supported = False - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - if self._resolving: - return self - elif self.parser is not posixpath: - raise UnsupportedOperation(self._unsupported_msg('resolve()')) - - def raise_error(*args): - raise OSError("Unsupported operation.") - - getcwd = raise_error - if strict or getattr(self.readlink, '_supported', True): - def lstat(path_str): - path = self.with_segments(path_str) - path._resolving = True - return path.stat(follow_symlinks=False) - - def readlink(path_str): - path = self.with_segments(path_str) - path._resolving = True - return str(path.readlink()) - else: - # If the user has *not* overridden the `readlink()` method, then - # symlinks are unsupported and (in non-strict mode) we can improve - # performance by not calling `path.lstat()`. - lstat = readlink = raise_error - - return self.with_segments(posixpath._realpath( - str(self.absolute()), strict, self.parser.sep, - getcwd=getcwd, lstat=lstat, readlink=readlink, - maxlinks=self._max_symlinks)) def symlink_to(self, target, target_is_directory=False): """ diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 2c48eeeda145d0..8c9049f15d5bf9 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1,3 +1,4 @@ +import collections import contextlib import io import os @@ -21,7 +22,7 @@ from test.support import os_helper from test.support.os_helper import TESTFN, FakePath from test.test_pathlib import test_pathlib_abc -from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows, needs_symlinks +from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows try: import fcntl @@ -55,6 +56,13 @@ def new_test(self): self.cls.replace = old_replace return new_test + +_tests_needing_symlinks = set() +def needs_symlinks(fn): + """Decorator that marks a test as requiring a path class that supports symlinks.""" + _tests_needing_symlinks.add(fn.__name__) + return fn + # # Tests for the pure classes. # @@ -533,6 +541,9 @@ class PathTest(test_pathlib_abc.DummyPathTest, PurePathTest): can_symlink = os_helper.can_symlink() def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') super().setUp() os.chmod(self.parser.join(self.base, 'dirE'), 0) @@ -693,6 +704,34 @@ def test_copy_file_preserve_metadata(self): if hasattr(source_st, 'st_flags'): self.assertEqual(source_st.st_flags, target_st.st_flags) + @needs_symlinks + def test_copy_file_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + @needs_symlinks + def test_copy_file_to_existing_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + @os_helper.skip_unless_xattr def test_copy_file_preserve_metadata_xattrs(self): base = self.cls(self.base) @@ -702,6 +741,118 @@ def test_copy_file_preserve_metadata_xattrs(self): source.copy(target, preserve_metadata=True) self.assertEqual(os.getxattr(target, b'user.foo'), b'42') + @needs_symlinks + def test_copy_symlink_follow_symlinks_true(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertFalse(target.is_symlink()) + self.assertEqual(source.read_text(), target.read_text()) + + @needs_symlinks + def test_copy_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.copy, source) + + @needs_symlinks + def test_copy_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_directory_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_into_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'linkB' / 'copyB' + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + self.assertFalse(target.exists()) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC') + target.symlink_to(base / 'fileA') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC' / 'dirD') + target.symlink_to(base / 'dirC') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'source' + target = base / 'target' + + source.mkdir() + source.joinpath('link').symlink_to('nonexistent') + + self.assertRaises(FileNotFoundError, source.copy, target) + + target2 = base / 'target2' + result = source.copy(target2, follow_symlinks=False) + self.assertEqual(result, target2) + self.assertTrue(target2.joinpath('link').is_symlink()) + self.assertEqual(target2.joinpath('link').readlink(), self.cls('nonexistent')) + @needs_symlinks def test_copy_link_preserve_metadata(self): base = self.cls(self.base) @@ -801,6 +952,54 @@ def test_copy_dir_preserve_metadata_xattrs(self): target_file = target.joinpath('dirD', 'fileD') self.assertEqual(os.getxattr(target_file, b'user.foo'), b'42') + @needs_symlinks + def test_move_file_symlink(self): + base = self.cls(self.base) + source = base / 'linkA' + source_readlink = source.readlink() + target = base / 'linkA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_file_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dir_symlink(self): + base = self.cls(self.base) + source = base / 'linkB' + source_readlink = source.readlink() + target = base / 'linkB_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_dir_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'brokenLink' + source_readlink = source.readlink() + target = base / 'brokenLink_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + @patch_replace def test_move_file_other_fs(self): self.test_move_file() @@ -858,9 +1057,41 @@ def test_move_into_other_os(self): def test_move_into_empty_name_other_os(self): self.test_move_into_empty_name() + @needs_symlinks + def test_complex_symlinks_absolute(self): + self._check_complex_symlinks(self.base) + + @needs_symlinks + def test_complex_symlinks_relative(self): + self._check_complex_symlinks('.') + + @needs_symlinks + def test_complex_symlinks_relative_dot_dot(self): + self._check_complex_symlinks(self.parser.join('dirA', '..')) + def _check_complex_symlinks(self, link0_target): - super()._check_complex_symlinks(link0_target) + # Test solving a non-looping chain of symlinks (issue #19887). + parser = self.parser P = self.cls(self.base) + P.joinpath('link1').symlink_to(parser.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(parser.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(parser.join('link2', 'link2'), target_is_directory=True) + P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) + + # Resolve absolute paths. + p = (P / 'link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + # Resolve relative paths. old_path = os.getcwd() os.chdir(self.base) @@ -880,6 +1111,118 @@ def _check_complex_symlinks(self, link0_target): finally: os.chdir(old_path) + def _check_resolve(self, p, expected, strict=True): + q = p.resolve(strict) + self.assertEqual(q, expected) + + # This can be used to check both relative and absolute resolutions. + _check_resolve_relative = _check_resolve_absolute = _check_resolve + + @needs_symlinks + def test_resolve_common(self): + P = self.cls + p = P(self.base, 'foo') + with self.assertRaises(OSError) as cm: + p.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ENOENT) + # Non-strict + parser = self.parser + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(self.base, 'foo')) + p = P(self.base, 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(self.base, 'foo', 'in', 'spam')) + p = P(self.base, '..', 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(parser.dirname(self.base), 'foo', 'in', 'spam')) + # These are all relative symlinks. + p = P(self.base, 'dirB', 'fileB') + self._check_resolve_relative(p, p) + p = P(self.base, 'linkA') + self._check_resolve_relative(p, P(self.base, 'fileA')) + p = P(self.base, 'dirA', 'linkC', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + p = P(self.base, 'dirB', 'linkD', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + # Non-strict + p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', + 'spam'), False) + p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', + 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + # Now create absolute symlinks. + d = self.tempdir() + P(self.base, 'dirA', 'linkX').symlink_to(d) + P(self.base, str(d), 'linkY').symlink_to(self.parser.join(self.base, 'dirB')) + p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') + self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) + # Non-strict + p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), + False) + p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + + @needs_symlinks + def test_resolve_dot(self): + # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ + parser = self.parser + p = self.cls(self.base) + p.joinpath('0').symlink_to('.', target_is_directory=True) + p.joinpath('1').symlink_to(parser.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(parser.join('1', '1'), target_is_directory=True) + q = p / '2' + self.assertEqual(q.resolve(strict=True), p) + r = q / '3' / '4' + self.assertRaises(FileNotFoundError, r.resolve, strict=True) + # Non-strict + self.assertEqual(r.resolve(strict=False), p / '3' / '4') + + def _check_symlink_loop(self, *args): + path = self.cls(*args) + with self.assertRaises(OSError) as cm: + path.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ELOOP) + + @needs_posix + @needs_symlinks + def test_resolve_loop(self): + # Loops with relative symlinks. + self.cls(self.base, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(self.base, 'linkX') + self.cls(self.base, 'linkY').symlink_to('linkY') + self._check_symlink_loop(self.base, 'linkY') + self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(self.base, 'linkZ') + # Non-strict + p = self.cls(self.base, 'linkZ', 'foo') + self.assertEqual(p.resolve(strict=False), p) + # Loops with absolute symlinks. + self.cls(self.base, 'linkU').symlink_to(self.parser.join(self.base, 'linkU/inside')) + self._check_symlink_loop(self.base, 'linkU') + self.cls(self.base, 'linkV').symlink_to(self.parser.join(self.base, 'linkV')) + self._check_symlink_loop(self.base, 'linkV') + self.cls(self.base, 'linkW').symlink_to(self.parser.join(self.base, 'linkW/../linkW')) + self._check_symlink_loop(self.base, 'linkW') + # Non-strict + q = self.cls(self.base, 'linkW', 'foo') + self.assertEqual(q.resolve(strict=False), q) + def test_resolve_nonexist_relative_issue38671(self): p = self.cls('non', 'exist') @@ -890,6 +1233,24 @@ def test_resolve_nonexist_relative_issue38671(self): finally: os.chdir(old_cwd) + @needs_symlinks + def test_readlink(self): + P = self.cls(self.base) + self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) + self.assertEqual((P / 'brokenLink').readlink(), + self.cls('non-existing')) + self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) + self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) + with self.assertRaises(OSError): + (P / 'fileA').readlink() + + @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") + def test_readlink_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.readlink(p) + @os_helper.skip_unless_working_chmod def test_chmod(self): p = self.cls(self.base) / 'fileA' @@ -991,6 +1352,41 @@ def test_group_no_follow_symlinks(self): self.assertEqual(expected_gid, gid_2) self.assertEqual(expected_name, link.group(follow_symlinks=False)) + @needs_symlinks + def test_delete_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir_ = tmp / 'dir' + dir_.mkdir() + link = tmp / 'link' + link.symlink_to(dir_) + link._delete() + self.assertTrue(dir_.exists()) + self.assertFalse(link.exists(follow_symlinks=False)) + + @needs_symlinks + def test_delete_inner_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + link1.symlink_to(dir2) + link2 = dir1 / 'link2' + link2.symlink_to(dir3) + link3 = dir1 / 'link3' + link3.symlink_to(file1) + # make sure symlinks are removed but not followed + dir1._delete() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") @os_helper.skip_if_dac_override @@ -1354,6 +1750,12 @@ def test_symlink_to_unsupported(self): with self.assertRaises(pathlib.UnsupportedOperation): q.symlink_to(p) + @needs_symlinks + def test_stat_no_follow_symlinks(self): + p = self.cls(self.base) / 'linkA' + st = p.stat() + self.assertNotEqual(st, p.stat(follow_symlinks=False)) + @needs_symlinks def test_lstat(self): p = self.cls(self.base)/ 'linkA' @@ -1433,6 +1835,15 @@ def test_passing_kwargs_errors(self): with self.assertRaises(TypeError): self.cls(foo="bar") + @needs_symlinks + def test_iterdir_symlink(self): + # __iter__ on a symlink to a directory. + P = self.cls + p = P(self.base, 'linkB') + paths = set(p.iterdir()) + expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } + self.assertEqual(paths, expected) + def test_glob_empty_pattern(self): p = self.cls('') with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): @@ -1493,6 +1904,25 @@ def test_glob_dot(self): self.assertEqual( set(P('.').glob('**/*/*')), {P("dirD/fileD")}) + # See https://github.com/WebAssembly/wasi-filesystem/issues/26 + @unittest.skipIf(is_wasi, "WASI resolution of '..' parts doesn't match POSIX") + def test_glob_dotdot(self): + # ".." is not special in globs. + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) + self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) + self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) + self.assertEqual(set(p.glob("dirA/../file*/..")), set()) + self.assertEqual(set(p.glob("../xyzzy")), set()) + if self.cls.parser is posixpath: + self.assertEqual(set(p.glob("xyzzy/..")), set()) + else: + # ".." segments are normalized first on Windows, so this path is stat()able. + self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) + self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) + def test_glob_inaccessible(self): P = self.cls p = P(self.base, "mydir1", "mydir2") @@ -1508,6 +1938,124 @@ def test_rglob_pathlike(self): self.assertEqual(expect, set(p.rglob(P(pattern)))) self.assertEqual(expect, set(p.rglob(FakePath(pattern)))) + @needs_symlinks + @unittest.skipIf(is_emscripten, "Hangs") + def test_glob_recurse_symlinks_common(self): + def _check(path, glob, expected): + actual = {path for path in path.glob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", []) + _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) + _check(p, "*A", ["dirA", "fileA", "linkA"]) + _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) + _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", + "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) + _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", + "dirB/linkD/..", "dirA/linkC/linkD/..", + "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", [ + "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", + "dirB/linkD/", "dirB/linkD/fileB", + "dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) + _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", + "dirB/linkD/..", "dirC/dirD/.."]) + _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "*/dirD/**/", ["dirC/dirD/"]) + + @needs_symlinks + @unittest.skipIf(is_emscripten, "Hangs") + def test_rglob_recurse_symlinks_common(self): + def _check(path, glob, expected): + actual = {path for path in path.rglob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "*/fileA", []) + _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB", + "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) + _check(p, "", ["", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) + + p = P(self.base, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + @needs_symlinks + def test_rglob_symlink_loop(self): + # Don't get fooled by symlink loops (Issue #26012). + P = self.cls + p = P(self.base) + given = set(p.rglob('*', recurse_symlinks=False)) + expect = {'brokenLink', + 'dirA', 'dirA/linkC', + 'dirB', 'dirB/fileB', 'dirB/linkD', + 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', + 'dirC/fileC', 'dirC/novel.txt', + 'dirE', + 'fileA', + 'linkA', + 'linkB', + 'brokenLinkLoop', + } + self.assertEqual(given, {p / x for x in expect}) + + @needs_symlinks + def test_glob_permissions(self): + # See bpo-38894 + P = self.cls + base = P(self.base) / 'permissions' + base.mkdir() + + for i in range(100): + link = base / f"link{i}" + if i % 2: + link.symlink_to(P(self.base, "dirE", "nonexistent")) + else: + link.symlink_to(P(self.base, "dirC"), target_is_directory=True) + + self.assertEqual(len(set(base.glob("*"))), 100) + self.assertEqual(len(set(base.glob("*/"))), 50) + self.assertEqual(len(set(base.glob("*/fileC"))), 50) + self.assertEqual(len(set(base.glob("*/file*"))), 50) + + @needs_symlinks + def test_glob_long_symlink(self): + # See gh-87695 + base = self.cls(self.base) / 'long_symlink' + base.mkdir() + bad_link = base / 'bad_link' + bad_link.symlink_to("bad" * 200) + self.assertEqual(sorted(base.glob('**/*')), [bad_link]) + @needs_posix def test_absolute_posix(self): P = self.cls @@ -1822,6 +2370,9 @@ class PathWalkTest(test_pathlib_abc.DummyPathWalkTest): can_symlink = PathTest.can_symlink def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') super().setUp() sub21_path= self.sub2_path / "SUB21" tmp5_path = sub21_path / "tmp3" @@ -1903,6 +2454,37 @@ def test_walk_above_recursion_limit(self): list(base.walk()) list(base.walk(top_down=False)) + @needs_symlinks + def test_walk_follow_symlinks(self): + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.link_path: + self.assertEqual(dirs, []) + self.assertEqual(files, ["tmp4"]) + break + else: + self.fail("Didn't follow symlink with follow_symlinks=True") + + @needs_symlinks + def test_walk_symlink_location(self): + # Tests whether symlinks end up in filenames or dirnames depending + # on the `follow_symlinks` argument. + walk_it = self.walk_path.walk(follow_symlinks=False) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", files) + break + else: + self.fail("symlink not found") + + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", dirs) + break + else: + self.fail("symlink not found") + @unittest.skipIf(os.name == 'nt', 'test requires a POSIX-compatible system') class PosixPathTest(PathTest, PurePosixPathTest): diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 00153e3f5e997e..bf9ae6cc8a2433 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -8,13 +8,11 @@ from pathlib._abc import UnsupportedOperation, ParserBase, PurePathBase, PathBase import posixpath -from test.support import is_wasi, is_emscripten from test.support.os_helper import TESTFN _tests_needing_posix = set() _tests_needing_windows = set() -_tests_needing_symlinks = set() def needs_posix(fn): @@ -27,11 +25,6 @@ def needs_windows(fn): _tests_needing_windows.add(fn.__name__) return fn -def needs_symlinks(fn): - """Decorator that marks a test as requiring a path class that supports symlinks.""" - _tests_needing_symlinks.add(fn.__name__) - return fn - class UnsupportedOperationTest(unittest.TestCase): def test_is_notimplemented(self): @@ -1369,7 +1362,6 @@ def test_unsupported_operation(self): self.assertRaises(e, p.glob, '*') self.assertRaises(e, p.rglob, '*') self.assertRaises(e, lambda: list(p.walk())) - self.assertRaises(e, p.absolute) self.assertRaises(e, p.expanduser) self.assertRaises(e, p.readlink) self.assertRaises(e, p.symlink_to, 'foo') @@ -1425,7 +1417,6 @@ class DummyPath(PathBase): _files = {} _directories = {} - _symlinks = {} def __eq__(self, other): if not isinstance(other, DummyPath): @@ -1439,16 +1430,11 @@ def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) def stat(self, *, follow_symlinks=True): - if follow_symlinks or self.name in ('', '.', '..'): - path = str(self.resolve(strict=True)) - else: - path = str(self.parent.resolve(strict=True) / self.name) + path = str(self).rstrip('/') if path in self._files: st_mode = stat.S_IFREG elif path in self._directories: st_mode = stat.S_IFDIR - elif path in self._symlinks: - st_mode = stat.S_IFLNK else: raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) return DummyPathStatResult(st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0) @@ -1457,10 +1443,7 @@ def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): if buffering != -1 and not (buffering == 0 and 'b' in mode): raise NotImplementedError - path_obj = self.resolve() - path = str(path_obj) - name = path_obj.name - parent = str(path_obj.parent) + path = str(self) if path in self._directories: raise IsADirectoryError(errno.EISDIR, "Is a directory", path) @@ -1471,6 +1454,7 @@ def open(self, mode='r', buffering=-1, encoding=None, raise FileNotFoundError(errno.ENOENT, "File not found", path) stream = io.BytesIO(self._files[path]) elif mode == 'w': + parent, name = posixpath.split(path) if parent not in self._directories: raise FileNotFoundError(errno.ENOENT, "File not found", parent) stream = DummyPathIO(self._files, path) @@ -1483,7 +1467,7 @@ def open(self, mode='r', buffering=-1, encoding=None, return stream def iterdir(self): - path = str(self.resolve()) + path = str(self).rstrip('/') if path in self._files: raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) elif path in self._directories: @@ -1492,9 +1476,9 @@ def iterdir(self): raise FileNotFoundError(errno.ENOENT, "File not found", path) def mkdir(self, mode=0o777, parents=False, exist_ok=False): - path = str(self.parent.resolve() / self.name) - parent = str(self.parent.resolve()) - if path in self._directories or path in self._symlinks: + path = str(self) + parent = str(self.parent) + if path in self._directories: if exist_ok: return else: @@ -1510,33 +1494,28 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): self.mkdir(mode, parents=False, exist_ok=exist_ok) def unlink(self, missing_ok=False): - path_obj = self.parent.resolve(strict=True) / self.name - path = str(path_obj) - name = path_obj.name - parent = str(path_obj.parent) + path = str(self) + name = self.name + parent = str(self.parent) if path in self._directories: raise IsADirectoryError(errno.EISDIR, "Is a directory", path) elif path in self._files: self._directories[parent].remove(name) del self._files[path] - elif path in self._symlinks: - self._directories[parent].remove(name) - del self._symlinks[path] elif not missing_ok: raise FileNotFoundError(errno.ENOENT, "File not found", path) def rmdir(self): - path_obj = self.parent.resolve(strict=True) / self.name - path = str(path_obj) - if path in self._files or path in self._symlinks: + path = str(self) + if path in self._files: raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) elif path not in self._directories: raise FileNotFoundError(errno.ENOENT, "File not found", path) elif self._directories[path]: raise OSError(errno.ENOTEMPTY, "Directory not empty", path) else: - name = path_obj.name - parent = str(path_obj.parent) + name = self.name + parent = str(self.parent) self._directories[parent].remove(name) del self._directories[path] @@ -1569,9 +1548,6 @@ class DummyPathTest(DummyPurePathTest): def setUp(self): super().setUp() - name = self.id().split('.')[-1] - if name in _tests_needing_symlinks and not self.can_symlink: - self.skipTest('requires symlinks') parser = self.cls.parser p = self.cls(self.base) p.mkdir(parents=True) @@ -1604,7 +1580,6 @@ def tearDown(self): cls = self.cls cls._files.clear() cls._directories.clear() - cls._symlinks.clear() def tempdir(self): path = self.cls(self.base).with_name('tmp-dirD') @@ -1730,101 +1705,6 @@ def test_copy_file(self): self.assertTrue(target.exists()) self.assertEqual(source.read_text(), target.read_text()) - @needs_symlinks - def test_copy_symlink_follow_symlinks_true(self): - base = self.cls(self.base) - source = base / 'linkA' - target = base / 'copyA' - result = source.copy(target) - self.assertEqual(result, target) - self.assertTrue(target.exists()) - self.assertFalse(target.is_symlink()) - self.assertEqual(source.read_text(), target.read_text()) - - @needs_symlinks - def test_copy_symlink_follow_symlinks_false(self): - base = self.cls(self.base) - source = base / 'linkA' - target = base / 'copyA' - result = source.copy(target, follow_symlinks=False) - self.assertEqual(result, target) - self.assertTrue(target.exists()) - self.assertTrue(target.is_symlink()) - self.assertEqual(source.readlink(), target.readlink()) - - @needs_symlinks - def test_copy_symlink_to_itself(self): - base = self.cls(self.base) - source = base / 'linkA' - self.assertRaises(OSError, source.copy, source) - - @needs_symlinks - def test_copy_symlink_to_existing_symlink(self): - base = self.cls(self.base) - source = base / 'copySource' - target = base / 'copyTarget' - source.symlink_to(base / 'fileA') - target.symlink_to(base / 'dirC') - self.assertRaises(OSError, source.copy, target) - self.assertRaises(OSError, source.copy, target, follow_symlinks=False) - - @needs_symlinks - def test_copy_symlink_to_existing_directory_symlink(self): - base = self.cls(self.base) - source = base / 'copySource' - target = base / 'copyTarget' - source.symlink_to(base / 'fileA') - target.symlink_to(base / 'dirC') - self.assertRaises(OSError, source.copy, target) - self.assertRaises(OSError, source.copy, target, follow_symlinks=False) - - @needs_symlinks - def test_copy_directory_symlink_follow_symlinks_false(self): - base = self.cls(self.base) - source = base / 'linkB' - target = base / 'copyA' - result = source.copy(target, follow_symlinks=False) - self.assertEqual(result, target) - self.assertTrue(target.exists()) - self.assertTrue(target.is_symlink()) - self.assertEqual(source.readlink(), target.readlink()) - - @needs_symlinks - def test_copy_directory_symlink_to_itself(self): - base = self.cls(self.base) - source = base / 'linkB' - self.assertRaises(OSError, source.copy, source) - self.assertRaises(OSError, source.copy, source, follow_symlinks=False) - - @needs_symlinks - def test_copy_directory_symlink_into_itself(self): - base = self.cls(self.base) - source = base / 'linkB' - target = base / 'linkB' / 'copyB' - self.assertRaises(OSError, source.copy, target) - self.assertRaises(OSError, source.copy, target, follow_symlinks=False) - self.assertFalse(target.exists()) - - @needs_symlinks - def test_copy_directory_symlink_to_existing_symlink(self): - base = self.cls(self.base) - source = base / 'copySource' - target = base / 'copyTarget' - source.symlink_to(base / 'dirC') - target.symlink_to(base / 'fileA') - self.assertRaises(FileExistsError, source.copy, target) - self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) - - @needs_symlinks - def test_copy_directory_symlink_to_existing_directory_symlink(self): - base = self.cls(self.base) - source = base / 'copySource' - target = base / 'copyTarget' - source.symlink_to(base / 'dirC' / 'dirD') - target.symlink_to(base / 'dirC') - self.assertRaises(FileExistsError, source.copy, target) - self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) - def test_copy_file_to_existing_file(self): base = self.cls(self.base) source = base / 'fileA' @@ -1840,34 +1720,6 @@ def test_copy_file_to_existing_directory(self): target = base / 'dirA' self.assertRaises(OSError, source.copy, target) - @needs_symlinks - def test_copy_file_to_existing_symlink(self): - base = self.cls(self.base) - source = base / 'dirB' / 'fileB' - target = base / 'linkA' - real_target = base / 'fileA' - result = source.copy(target) - self.assertEqual(result, target) - self.assertTrue(target.exists()) - self.assertTrue(target.is_symlink()) - self.assertTrue(real_target.exists()) - self.assertFalse(real_target.is_symlink()) - self.assertEqual(source.read_text(), real_target.read_text()) - - @needs_symlinks - def test_copy_file_to_existing_symlink_follow_symlinks_false(self): - base = self.cls(self.base) - source = base / 'dirB' / 'fileB' - target = base / 'linkA' - real_target = base / 'fileA' - result = source.copy(target, follow_symlinks=False) - self.assertEqual(result, target) - self.assertTrue(target.exists()) - self.assertTrue(target.is_symlink()) - self.assertTrue(real_target.exists()) - self.assertFalse(real_target.is_symlink()) - self.assertEqual(source.read_text(), real_target.read_text()) - def test_copy_file_empty(self): base = self.cls(self.base) source = base / 'empty' @@ -1985,23 +1837,6 @@ def test_copy_dir_into_itself(self): self.assertRaises(OSError, source.copy, target, follow_symlinks=False) self.assertFalse(target.exists()) - @needs_symlinks - def test_copy_dangling_symlink(self): - base = self.cls(self.base) - source = base / 'source' - target = base / 'target' - - source.mkdir() - source.joinpath('link').symlink_to('nonexistent') - - self.assertRaises(FileNotFoundError, source.copy, target) - - target2 = base / 'target2' - result = source.copy(target2, follow_symlinks=False) - self.assertEqual(result, target2) - self.assertTrue(target2.joinpath('link').is_symlink()) - self.assertEqual(target2.joinpath('link').readlink(), self.cls('nonexistent')) - def test_copy_into(self): base = self.cls(self.base) source = base / 'fileA' @@ -2087,54 +1922,6 @@ def test_move_dir_into_itself(self): self.assertTrue(source.exists()) self.assertFalse(target.exists()) - @needs_symlinks - def test_move_file_symlink(self): - base = self.cls(self.base) - source = base / 'linkA' - source_readlink = source.readlink() - target = base / 'linkA_moved' - result = source.move(target) - self.assertEqual(result, target) - self.assertFalse(source.exists()) - self.assertTrue(target.is_symlink()) - self.assertEqual(source_readlink, target.readlink()) - - @needs_symlinks - def test_move_file_symlink_to_itself(self): - base = self.cls(self.base) - source = base / 'linkA' - self.assertRaises(OSError, source.move, source) - - @needs_symlinks - def test_move_dir_symlink(self): - base = self.cls(self.base) - source = base / 'linkB' - source_readlink = source.readlink() - target = base / 'linkB_moved' - result = source.move(target) - self.assertEqual(result, target) - self.assertFalse(source.exists()) - self.assertTrue(target.is_symlink()) - self.assertEqual(source_readlink, target.readlink()) - - @needs_symlinks - def test_move_dir_symlink_to_itself(self): - base = self.cls(self.base) - source = base / 'linkB' - self.assertRaises(OSError, source.move, source) - - @needs_symlinks - def test_move_dangling_symlink(self): - base = self.cls(self.base) - source = base / 'brokenLink' - source_readlink = source.readlink() - target = base / 'brokenLink_moved' - result = source.move(target) - self.assertEqual(result, target) - self.assertFalse(source.exists()) - self.assertTrue(target.is_symlink()) - self.assertEqual(source_readlink, target.readlink()) - def test_move_into(self): base = self.cls(self.base) source = base / 'fileA' @@ -2161,15 +1948,6 @@ def test_iterdir(self): expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] self.assertEqual(paths, { P(self.base, q) for q in expected }) - @needs_symlinks - def test_iterdir_symlink(self): - # __iter__ on a symlink to a directory. - P = self.cls - p = P(self.base, 'linkB') - paths = set(p.iterdir()) - expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } - self.assertEqual(paths, expected) - def test_iterdir_nodir(self): # __iter__ on something that is not a directory. p = self.cls(self.base, 'fileA') @@ -2196,7 +1974,6 @@ def test_scandir(self): if entry.name != 'brokenLinkLoop': self.assertEqual(entry.is_dir(), child.is_dir()) - def test_glob_common(self): def _check(glob, expected): self.assertEqual(set(glob), { P(self.base, q) for q in expected }) @@ -2250,8 +2027,6 @@ def test_glob_empty_pattern(self): P = self.cls p = P(self.base) self.assertEqual(list(p.glob("")), [p]) - self.assertEqual(list(p.glob(".")), [p / "."]) - self.assertEqual(list(p.glob("./")), [p / "./"]) def test_glob_case_sensitive(self): P = self.cls @@ -2265,44 +2040,6 @@ def _check(path, pattern, case_sensitive, expected): _check(path, "dirb/file*", True, []) _check(path, "dirb/file*", False, ["dirB/fileB"]) - @needs_symlinks - @unittest.skipIf(is_emscripten, "Hangs") - def test_glob_recurse_symlinks_common(self): - def _check(path, glob, expected): - actual = {path for path in path.glob(glob, recurse_symlinks=True) - if path.parts.count("linkD") <= 1} # exclude symlink loop. - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) - _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) - _check(p, "dir*/**", [ - "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", - "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", - "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", - "dirE/"]) - _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", - "dirB/linkD/..", "dirA/linkC/linkD/..", - "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**", [ - "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", - "dirB/linkD/", "dirB/linkD/fileB", - "dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) - _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", - "dirB/linkD/..", "dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) - _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "*/dirD/**/", ["dirC/dirD/"]) - def test_rglob_recurse_symlinks_false(self): def _check(path, glob, expected): actual = set(path.rglob(glob, recurse_symlinks=False)) @@ -2361,252 +2098,6 @@ def test_rglob_windows(self): self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) - @needs_symlinks - @unittest.skipIf(is_emscripten, "Hangs") - def test_rglob_recurse_symlinks_common(self): - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, recurse_symlinks=True) - if path.parts.count("linkD") <= 1} # exclude symlink loop. - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", - "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) - _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", - "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) - _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", - "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB", - "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) - _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) - _check(p, "", ["", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) - - p = P(self.base, "dirC") - _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD/"]) - _check(p, "", ["dirC/", "dirC/dirD/"]) - # gh-91616, a re module regression - _check(p, "*.txt", ["dirC/novel.txt"]) - _check(p, "*.*", ["dirC/novel.txt"]) - - @needs_symlinks - def test_rglob_symlink_loop(self): - # Don't get fooled by symlink loops (Issue #26012). - P = self.cls - p = P(self.base) - given = set(p.rglob('*', recurse_symlinks=False)) - expect = {'brokenLink', - 'dirA', 'dirA/linkC', - 'dirB', 'dirB/fileB', 'dirB/linkD', - 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', - 'dirC/fileC', 'dirC/novel.txt', - 'dirE', - 'fileA', - 'linkA', - 'linkB', - 'brokenLinkLoop', - } - self.assertEqual(given, {p / x for x in expect}) - - # See https://github.com/WebAssembly/wasi-filesystem/issues/26 - @unittest.skipIf(is_wasi, "WASI resolution of '..' parts doesn't match POSIX") - def test_glob_dotdot(self): - # ".." is not special in globs. - P = self.cls - p = P(self.base) - self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) - self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) - self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) - self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) - self.assertEqual(set(p.glob("dirA/../file*/..")), set()) - self.assertEqual(set(p.glob("../xyzzy")), set()) - if self.cls.parser is posixpath: - self.assertEqual(set(p.glob("xyzzy/..")), set()) - else: - # ".." segments are normalized first on Windows, so this path is stat()able. - self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) - - @needs_symlinks - def test_glob_permissions(self): - # See bpo-38894 - P = self.cls - base = P(self.base) / 'permissions' - base.mkdir() - - for i in range(100): - link = base / f"link{i}" - if i % 2: - link.symlink_to(P(self.base, "dirE", "nonexistent")) - else: - link.symlink_to(P(self.base, "dirC"), target_is_directory=True) - - self.assertEqual(len(set(base.glob("*"))), 100) - self.assertEqual(len(set(base.glob("*/"))), 50) - self.assertEqual(len(set(base.glob("*/fileC"))), 50) - self.assertEqual(len(set(base.glob("*/file*"))), 50) - - @needs_symlinks - def test_glob_long_symlink(self): - # See gh-87695 - base = self.cls(self.base) / 'long_symlink' - base.mkdir() - bad_link = base / 'bad_link' - bad_link.symlink_to("bad" * 200) - self.assertEqual(sorted(base.glob('**/*')), [bad_link]) - - @needs_posix - def test_absolute_posix(self): - P = self.cls - # The default implementation uses '/' as the current directory - self.assertEqual(str(P('').absolute()), '/') - self.assertEqual(str(P('a').absolute()), '/a') - self.assertEqual(str(P('a/b').absolute()), '/a/b') - - self.assertEqual(str(P('/').absolute()), '/') - self.assertEqual(str(P('/a').absolute()), '/a') - self.assertEqual(str(P('/a/b').absolute()), '/a/b') - - # '//'-prefixed absolute path (supported by POSIX). - self.assertEqual(str(P('//').absolute()), '//') - self.assertEqual(str(P('//a').absolute()), '//a') - self.assertEqual(str(P('//a/b').absolute()), '//a/b') - - @needs_symlinks - def test_readlink(self): - P = self.cls(self.base) - self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) - self.assertEqual((P / 'brokenLink').readlink(), - self.cls('non-existing')) - self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) - self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) - with self.assertRaises(OSError): - (P / 'fileA').readlink() - - @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") - def test_readlink_unsupported(self): - P = self.cls(self.base) - p = P / 'fileA' - with self.assertRaises(UnsupportedOperation): - q.readlink(p) - - def _check_resolve(self, p, expected, strict=True): - q = p.resolve(strict) - self.assertEqual(q, expected) - - # This can be used to check both relative and absolute resolutions. - _check_resolve_relative = _check_resolve_absolute = _check_resolve - - @needs_symlinks - def test_resolve_common(self): - P = self.cls - p = P(self.base, 'foo') - with self.assertRaises(OSError) as cm: - p.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ENOENT) - # Non-strict - parser = self.parser - self.assertEqualNormCase(str(p.resolve(strict=False)), - parser.join(self.base, 'foo')) - p = P(self.base, 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - parser.join(self.base, 'foo', 'in', 'spam')) - p = P(self.base, '..', 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - parser.join(parser.dirname(self.base), 'foo', 'in', 'spam')) - # These are all relative symlinks. - p = P(self.base, 'dirB', 'fileB') - self._check_resolve_relative(p, p) - p = P(self.base, 'linkA') - self._check_resolve_relative(p, P(self.base, 'fileA')) - p = P(self.base, 'dirA', 'linkC', 'fileB') - self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) - p = P(self.base, 'dirB', 'linkD', 'fileB') - self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) - # Non-strict - p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', - 'spam'), False) - p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if self.cls.parser is not posixpath: - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', - 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) - # Now create absolute symlinks. - d = self.tempdir() - P(self.base, 'dirA', 'linkX').symlink_to(d) - P(self.base, str(d), 'linkY').symlink_to(self.parser.join(self.base, 'dirB')) - p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') - self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) - # Non-strict - p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), - False) - p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if self.cls.parser is not posixpath: - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) - - @needs_symlinks - def test_resolve_dot(self): - # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - parser = self.parser - p = self.cls(self.base) - p.joinpath('0').symlink_to('.', target_is_directory=True) - p.joinpath('1').symlink_to(parser.join('0', '0'), target_is_directory=True) - p.joinpath('2').symlink_to(parser.join('1', '1'), target_is_directory=True) - q = p / '2' - self.assertEqual(q.resolve(strict=True), p) - r = q / '3' / '4' - self.assertRaises(FileNotFoundError, r.resolve, strict=True) - # Non-strict - self.assertEqual(r.resolve(strict=False), p / '3' / '4') - - def _check_symlink_loop(self, *args): - path = self.cls(*args) - with self.assertRaises(OSError) as cm: - path.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ELOOP) - - @needs_posix - @needs_symlinks - def test_resolve_loop(self): - # Loops with relative symlinks. - self.cls(self.base, 'linkX').symlink_to('linkX/inside') - self._check_symlink_loop(self.base, 'linkX') - self.cls(self.base, 'linkY').symlink_to('linkY') - self._check_symlink_loop(self.base, 'linkY') - self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') - self._check_symlink_loop(self.base, 'linkZ') - # Non-strict - p = self.cls(self.base, 'linkZ', 'foo') - self.assertEqual(p.resolve(strict=False), p) - # Loops with absolute symlinks. - self.cls(self.base, 'linkU').symlink_to(self.parser.join(self.base, 'linkU/inside')) - self._check_symlink_loop(self.base, 'linkU') - self.cls(self.base, 'linkV').symlink_to(self.parser.join(self.base, 'linkV')) - self._check_symlink_loop(self.base, 'linkV') - self.cls(self.base, 'linkW').symlink_to(self.parser.join(self.base, 'linkW/../linkW')) - self._check_symlink_loop(self.base, 'linkW') - # Non-strict - q = self.cls(self.base, 'linkW', 'foo') - self.assertEqual(q.resolve(strict=False), q) - def test_stat(self): statA = self.cls(self.base).joinpath('fileA').stat() statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() @@ -2627,12 +2118,6 @@ def test_stat(self): self.assertEqual(statA.st_dev, statC.st_dev) # other attributes not used by pathlib. - @needs_symlinks - def test_stat_no_follow_symlinks(self): - p = self.cls(self.base) / 'linkA' - st = p.stat() - self.assertNotEqual(st, p.stat(follow_symlinks=False)) - def test_stat_no_follow_symlinks_nosymlink(self): p = self.cls(self.base) / 'fileA' st = p.stat() @@ -2760,41 +2245,6 @@ def test_is_char_device_false(self): self.assertIs((P / 'fileA\udfff').is_char_device(), False) self.assertIs((P / 'fileA\x00').is_char_device(), False) - def _check_complex_symlinks(self, link0_target): - # Test solving a non-looping chain of symlinks (issue #19887). - parser = self.parser - P = self.cls(self.base) - P.joinpath('link1').symlink_to(parser.join('link0', 'link0'), target_is_directory=True) - P.joinpath('link2').symlink_to(parser.join('link1', 'link1'), target_is_directory=True) - P.joinpath('link3').symlink_to(parser.join('link2', 'link2'), target_is_directory=True) - P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) - - # Resolve absolute paths. - p = (P / 'link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = (P / 'link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = (P / 'link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = (P / 'link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - - @needs_symlinks - def test_complex_symlinks_absolute(self): - self._check_complex_symlinks(self.base) - - @needs_symlinks - def test_complex_symlinks_relative(self): - self._check_complex_symlinks('.') - - @needs_symlinks - def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(self.parser.join('dirA', '..')) - def test_unlink(self): p = self.cls(self.base) / 'fileA' p.unlink() @@ -2838,41 +2288,6 @@ def test_delete_dir(self): self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'fileC').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'novel.txt').stat) - @needs_symlinks - def test_delete_symlink(self): - tmp = self.cls(self.base, 'delete') - tmp.mkdir() - dir_ = tmp / 'dir' - dir_.mkdir() - link = tmp / 'link' - link.symlink_to(dir_) - link._delete() - self.assertTrue(dir_.exists()) - self.assertFalse(link.exists(follow_symlinks=False)) - - @needs_symlinks - def test_delete_inner_symlink(self): - tmp = self.cls(self.base, 'delete') - tmp.mkdir() - dir1 = tmp / 'dir1' - dir2 = dir1 / 'dir2' - dir3 = tmp / 'dir3' - for d in dir1, dir2, dir3: - d.mkdir() - file1 = tmp / 'file1' - file1.write_text('foo') - link1 = dir1 / 'link1' - link1.symlink_to(dir2) - link2 = dir1 / 'link2' - link2.symlink_to(dir3) - link3 = dir1 / 'link3' - link3.symlink_to(file1) - # make sure symlinks are removed but not followed - dir1._delete() - self.assertFalse(dir1.exists()) - self.assertTrue(dir3.exists()) - self.assertTrue(file1.exists()) - def test_delete_missing(self): tmp = self.cls(self.base, 'delete') tmp.mkdir() @@ -2887,9 +2302,6 @@ class DummyPathWalkTest(unittest.TestCase): can_symlink = False def setUp(self): - name = self.id().split('.')[-1] - if name in _tests_needing_symlinks and not self.can_symlink: - self.skipTest('requires symlinks') # Build: # TESTFN/ # TEST1/ a file kid and two directory kids @@ -3002,70 +2414,6 @@ def test_walk_bottom_up(self): raise AssertionError(f"Unexpected path: {path}") self.assertTrue(seen_testfn) - @needs_symlinks - def test_walk_follow_symlinks(self): - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.link_path: - self.assertEqual(dirs, []) - self.assertEqual(files, ["tmp4"]) - break - else: - self.fail("Didn't follow symlink with follow_symlinks=True") - - @needs_symlinks - def test_walk_symlink_location(self): - # Tests whether symlinks end up in filenames or dirnames depending - # on the `follow_symlinks` argument. - walk_it = self.walk_path.walk(follow_symlinks=False) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", files) - break - else: - self.fail("symlink not found") - - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", dirs) - break - else: - self.fail("symlink not found") - - -class DummyPathWithSymlinks(DummyPath): - __slots__ = () - - # Reduce symlink traversal limit to make tests run faster. - _max_symlinks = 20 - - def readlink(self): - path = str(self.parent.resolve() / self.name) - if path in self._symlinks: - return self.with_segments(self._symlinks[path][0]) - elif path in self._files or path in self._directories: - raise OSError(errno.EINVAL, "Not a symlink", path) - else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - - def symlink_to(self, target, target_is_directory=False): - path = str(self.parent.resolve() / self.name) - parent = str(self.parent.resolve()) - if path in self._symlinks: - raise FileExistsError(errno.EEXIST, "File exists", path) - self._directories[parent].add(self.name) - self._symlinks[path] = str(target), target_is_directory - - -class DummyPathWithSymlinksTest(DummyPathTest): - cls = DummyPathWithSymlinks - can_symlink = True - - -class DummyPathWithSymlinksWalkTest(DummyPathWalkTest): - cls = DummyPathWithSymlinks - can_symlink = True if __name__ == "__main__": From 72dca6c4eda0d63ee35a0aa619ae931ab226bef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Srinivas=20Reddy=20Thatiparthy=20=28=E0=B0=A4=E0=B0=BE?= =?UTF-8?q?=E0=B0=9F=E0=B0=BF=E0=B0=AA=E0=B0=B0=E0=B1=8D=E0=B0=A4=E0=B0=BF?= =?UTF-8?q?=20=E0=B0=B6=E0=B1=8D=E0=B0=B0=E0=B1=80=E0=B0=A8=E0=B0=BF?= =?UTF-8?q?=E0=B0=B5=E0=B0=BE=E0=B0=B8=E0=B1=8D=20=20=E0=B0=B0=E0=B1=86?= =?UTF-8?q?=E0=B0=A1=E0=B1=8D=E0=B0=A1=E0=B0=BF=29?= Date: Sat, 7 Dec 2024 15:42:45 +0530 Subject: [PATCH 131/427] gh-119786: fix typo in `InternalDocs/garbage_collector.md` (#127687) --- InternalDocs/garbage_collector.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 4761f78f3593e3..394e4ef075f55e 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -518,7 +518,7 @@ Then the above algorithm is repeated, starting from step 2. Determining how much work to do ------------------------------- -We need to do a certain amount of work to enusre that garbage is collected, +We need to do a certain amount of work to ensure that garbage is collected, but doing too much work slows down execution. To work out how much work we need to do, consider a heap with `L` live objects From 27d0d2141319d82709eb09ba20065df3e1714fab Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:13:49 +0000 Subject: [PATCH 132/427] Give `poplib.POP3.rpop` a proper docstring (#127370) Previously `poplib.POP3.rpop` had a "Not sure what this does" docstring, now it has been fixed. --- Lib/poplib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/poplib.py b/Lib/poplib.py index 1a1629d175b6d9..beb93a0d57cf93 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -309,7 +309,7 @@ def close(self): # optional commands: def rpop(self, user): - """Not sure what this does.""" + """Send RPOP command to access the mailbox with an alternate user.""" return self._shortcmd('RPOP %s' % user) From 79b7cab50a3292a1c01466cf0e69fb7b4e56cfb1 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 7 Dec 2024 17:58:42 +0000 Subject: [PATCH 133/427] GH-127090: Fix `urllib.response.addinfourl.url` value for opened `file:` URIs (#127091) The canonical `file:` URL (as generated by `pathname2url()`) is now used as the `url` attribute of the returned `addinfourl` object. The `addinfourl.url` attribute reflects the resolved URL for both `file:` or `http[s]:` URLs now. --- Lib/test/test_urllib.py | 11 ++++--- Lib/test/test_urllib2.py | 31 ++++++++----------- Lib/test/test_urllib2net.py | 3 +- Lib/urllib/request.py | 5 +-- ...-11-21-06-03-46.gh-issue-127090.yUYwdh.rst | 3 ++ 5 files changed, 25 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 00e46990c406ac..042d3b35b77022 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -156,7 +156,7 @@ def test_headers(self): self.assertIsInstance(self.returned_obj.headers, email.message.Message) def test_url(self): - self.assertEqual(self.returned_obj.url, "file://" + self.quoted_pathname) + self.assertEqual(self.returned_obj.url, "file:" + self.quoted_pathname) def test_status(self): self.assertIsNone(self.returned_obj.status) @@ -165,7 +165,7 @@ def test_info(self): self.assertIsInstance(self.returned_obj.info(), email.message.Message) def test_geturl(self): - self.assertEqual(self.returned_obj.geturl(), "file://" + self.quoted_pathname) + self.assertEqual(self.returned_obj.geturl(), "file:" + self.quoted_pathname) def test_getcode(self): self.assertIsNone(self.returned_obj.getcode()) @@ -471,11 +471,14 @@ def test_missing_localfile(self): def test_file_notexists(self): fd, tmp_file = tempfile.mkstemp() - tmp_fileurl = 'file://localhost/' + tmp_file.replace(os.path.sep, '/') + tmp_file_canon_url = 'file:' + urllib.request.pathname2url(tmp_file) + parsed = urllib.parse.urlsplit(tmp_file_canon_url) + tmp_fileurl = parsed._replace(netloc='localhost').geturl() try: self.assertTrue(os.path.exists(tmp_file)) with urllib.request.urlopen(tmp_fileurl) as fobj: self.assertTrue(fobj) + self.assertEqual(fobj.url, tmp_file_canon_url) finally: os.close(fd) os.unlink(tmp_file) @@ -609,7 +612,7 @@ def tearDown(self): def constructLocalFileUrl(self, filePath): filePath = os.path.abspath(filePath) - return "file://%s" % urllib.request.pathname2url(filePath) + return "file:" + urllib.request.pathname2url(filePath) def createNewTempFile(self, data=b""): """Creates a new temporary file containing the specified data, diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 99ad11cf0552eb..4a9e653515be5b 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -23,7 +23,7 @@ _proxy_bypass_winreg_override, _proxy_bypass_macosx_sysconf, AbstractDigestAuthHandler) -from urllib.parse import urlparse +from urllib.parse import urlsplit import urllib.error import http.client @@ -717,14 +717,6 @@ def test_processors(self): self.assertIsInstance(args[1], MockResponse) -def sanepathname2url(path): - urlpath = urllib.request.pathname2url(path) - if os.name == "nt" and urlpath.startswith("///"): - urlpath = urlpath[2:] - # XXX don't ask me about the mac... - return urlpath - - class HandlerTests(unittest.TestCase): def test_ftp(self): @@ -818,19 +810,22 @@ def test_file(self): o = h.parent = MockOpener() TESTFN = os_helper.TESTFN - urlpath = sanepathname2url(os.path.abspath(TESTFN)) towrite = b"hello, world\n" + canonurl = 'file:' + urllib.request.pathname2url(os.path.abspath(TESTFN)) + parsed = urlsplit(canonurl) + if parsed.netloc: + raise unittest.SkipTest("non-local working directory") urls = [ - "file://localhost%s" % urlpath, - "file://%s" % urlpath, - "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), + canonurl, + parsed._replace(netloc='localhost').geturl(), + parsed._replace(netloc=socket.gethostbyname('localhost')).geturl(), ] try: localaddr = socket.gethostbyname(socket.gethostname()) except socket.gaierror: localaddr = '' if localaddr: - urls.append("file://%s%s" % (localaddr, urlpath)) + urls.append(parsed._replace(netloc=localaddr).geturl()) for url in urls: f = open(TESTFN, "wb") @@ -855,10 +850,10 @@ def test_file(self): self.assertEqual(headers["Content-type"], "text/plain") self.assertEqual(headers["Content-length"], "13") self.assertEqual(headers["Last-modified"], modified) - self.assertEqual(respurl, url) + self.assertEqual(respurl, canonurl) for url in [ - "file://localhost:80%s" % urlpath, + parsed._replace(netloc='localhost:80').geturl(), "file:///file_does_not_exist.txt", "file://not-a-local-host.com//dir/file.txt", "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), @@ -1156,13 +1151,13 @@ def test_full_url_setter(self): r = Request('http://example.com') for url in urls: r.full_url = url - parsed = urlparse(url) + parsed = urlsplit(url) self.assertEqual(r.get_full_url(), url) # full_url setter uses splittag to split into components. # splittag sets the fragment as None while urlparse sets it to '' self.assertEqual(r.fragment or '', parsed.fragment) - self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) + self.assertEqual(urlsplit(r.get_full_url()).query, parsed.query) def test_full_url_deleter(self): r = Request('http://www.example.com') diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index f0874d8d3ce463..b84290a7368c29 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -4,7 +4,6 @@ from test.support import os_helper from test.support import socket_helper from test.support import ResourceDenied -from test.test_urllib2 import sanepathname2url import os import socket @@ -151,7 +150,7 @@ def test_file(self): f.write('hi there\n') f.close() urls = [ - 'file:' + sanepathname2url(os.path.abspath(TESTFN)), + 'file:' + urllib.request.pathname2url(os.path.abspath(TESTFN)), ('file:///nonsensename/etc/passwd', None, urllib.error.URLError), ] diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 1fcaa89188188d..7ef85431b718ad 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1488,10 +1488,7 @@ def open_local_file(self, req): host, port = _splitport(host) if not host or \ (not port and _safe_gethostbyname(host) in self.get_names()): - if host: - origurl = 'file://' + host + filename - else: - origurl = 'file://' + filename + origurl = 'file:' + pathname2url(localfile) return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: raise URLError(exp, exp.filename) diff --git a/Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst b/Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst new file mode 100644 index 00000000000000..8efe563f443774 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst @@ -0,0 +1,3 @@ +Fix value of :attr:`urllib.response.addinfourl.url` for ``file:`` URLs that +express relative paths and absolute Windows paths. The canonical URL generated +by :func:`urllib.request.pathname2url` is now used. From 70154855cf698560dd9a5e484a649839cd68dc7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Sun, 8 Dec 2024 05:57:22 +0000 Subject: [PATCH 134/427] GH-126789: fix some sysconfig data on late site initializations (#127729) --- Lib/sysconfig/__init__.py | 18 +++++++++---- Lib/test/test_sysconfig.py | 25 +++++++++++++++++++ ...-12-07-23-06-44.gh-issue-126789.4dqfV1.rst | 5 ++++ 3 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index ad86609016e478..ed7b6a335d01d4 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -173,9 +173,7 @@ def joinuser(*args): _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' -_PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) -_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) # Mutex guarding initialization of _CONFIG_VARS. _CONFIG_VARS_LOCK = threading.RLock() @@ -473,8 +471,8 @@ def _init_config_vars(): global _CONFIG_VARS _CONFIG_VARS = {} - prefix = _PREFIX - exec_prefix = _EXEC_PREFIX + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) base_prefix = _BASE_PREFIX base_exec_prefix = _BASE_EXEC_PREFIX @@ -564,9 +562,19 @@ def get_config_vars(*args): With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ + global _CONFIG_VARS_INITIALIZED # Avoid claiming the lock once initialization is complete. - if not _CONFIG_VARS_INITIALIZED: + if _CONFIG_VARS_INITIALIZED: + # GH-126789: If sys.prefix or sys.exec_prefix were updated, invalidate the cache. + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + if _CONFIG_VARS['prefix'] != prefix or _CONFIG_VARS['exec_prefix'] != exec_prefix: + with _CONFIG_VARS_LOCK: + _CONFIG_VARS_INITIALIZED = False + _init_config_vars() + else: + # Initialize the config_vars cache. with _CONFIG_VARS_LOCK: # Test again with the lock held to avoid races. Note that # we test _CONFIG_VARS here, not _CONFIG_VARS_INITIALIZED, diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 0df1a67ea2b720..ce504dc21af85f 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -53,6 +53,8 @@ def setUp(self): os.uname = self._get_uname # saving the environment self.name = os.name + self.prefix = sys.prefix + self.exec_prefix = sys.exec_prefix self.platform = sys.platform self.version = sys.version self._framework = sys._framework @@ -77,6 +79,8 @@ def tearDown(self): else: del os.uname os.name = self.name + sys.prefix = self.prefix + sys.exec_prefix = self.exec_prefix sys.platform = self.platform sys.version = self.version sys._framework = self._framework @@ -653,6 +657,27 @@ def test_sysconfigdata_json(self): self.assertEqual(system_config_vars, json_config_vars) + def test_sysconfig_config_vars_no_prefix_cache(self): + sys.prefix = 'prefix-AAA' + sys.exec_prefix = 'exec-prefix-AAA' + + config_vars = sysconfig.get_config_vars() + + self.assertEqual(config_vars['prefix'], sys.prefix) + self.assertEqual(config_vars['base'], sys.prefix) + self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) + self.assertEqual(config_vars['platbase'], sys.exec_prefix) + + sys.prefix = 'prefix-BBB' + sys.exec_prefix = 'exec-prefix-BBB' + + config_vars = sysconfig.get_config_vars() + + self.assertEqual(config_vars['prefix'], sys.prefix) + self.assertEqual(config_vars['base'], sys.prefix) + self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) + self.assertEqual(config_vars['platbase'], sys.exec_prefix) + class MakefileTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst b/Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst new file mode 100644 index 00000000000000..417e9ac986f27a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst @@ -0,0 +1,5 @@ +Fixed :func:`sysconfig.get_config_vars`, :func:`sysconfig.get_paths`, and +siblings, returning outdated cached data if the value of :data:`sys.prefix` +or :data:`sys.exec_prefix` changes. Overwriting :data:`sys.prefix` or +:data:`sys.exec_prefix` still is discouraged, as that might break other +parts of the code. From 1503fc8f88d4903e61f76a78a30bcd581b0ee0cd Mon Sep 17 00:00:00 2001 From: Apostol Fet <90645107+ApostolFet@users.noreply.github.com> Date: Sun, 8 Dec 2024 13:05:15 +0300 Subject: [PATCH 135/427] gh-127610: Added validation for more than one var-positional and var-keyword parameters in inspect.Signature (GH-127657) --- Lib/inspect.py | 8 ++++++++ Lib/test/test_inspect/test_inspect.py | 11 +++++++++++ Misc/ACKS | 1 + .../2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst | 3 +++ 4 files changed, 23 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index e3f74e9f047eaf..b7d8271f8a471f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2943,11 +2943,19 @@ def __init__(self, parameters=None, *, return_annotation=_empty, params = OrderedDict() top_kind = _POSITIONAL_ONLY seen_default = False + seen_var_parameters = set() for param in parameters: kind = param.kind name = param.name + if kind in (_VAR_POSITIONAL, _VAR_KEYWORD): + if kind in seen_var_parameters: + msg = f'more than one {kind.description} parameter' + raise ValueError(msg) + + seen_var_parameters.add(kind) + if kind < top_kind: msg = ( 'wrong parameter order: {} parameter before {} ' diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index a92627a4d60f68..1ecf18bf49fa7e 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -2992,6 +2992,17 @@ def test2(pod=42, /): with self.assertRaisesRegex(ValueError, 'follows default argument'): S((pkd, pk)) + second_args = args.replace(name="second_args") + with self.assertRaisesRegex(ValueError, 'more than one variadic positional parameter'): + S((args, second_args)) + + with self.assertRaisesRegex(ValueError, 'more than one variadic positional parameter'): + S((args, ko, second_args)) + + second_kwargs = kwargs.replace(name="second_kwargs") + with self.assertRaisesRegex(ValueError, 'more than one variadic keyword parameter'): + S((kwargs, second_kwargs)) + def test_signature_object_pickle(self): def foo(a, b, *, c:1={}, **kw) -> {42:'ham'}: pass foo_partial = functools.partial(foo, a=1) diff --git a/Misc/ACKS b/Misc/ACKS index 913f7c8ecf5f1e..086930666822ad 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -24,6 +24,7 @@ Eitan Adler Anton Afanasyev Ali Afshar Nitika Agarwal +Maxim Ageev Anjani Agrawal Pablo S. Blum de Aguiar Jim Ahlstrom diff --git a/Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst b/Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst new file mode 100644 index 00000000000000..58769029d79977 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst @@ -0,0 +1,3 @@ +Added validation for more than one var-positional or +var-keyword parameters in :class:`inspect.Signature`. +Patch by Maxim Ageev. From 8fa5ecec01337215bc7baa62c9c16488ecd854fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 8 Dec 2024 14:47:22 +0100 Subject: [PATCH 136/427] gh-123378: fix post-merge typos in comments and NEWS (#127739) --- .../C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst | 2 +- Objects/exceptions.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst b/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst index 2cfb8b8a1e245a..7254a04f61843d 100644 --- a/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst +++ b/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst @@ -1,5 +1,5 @@ Ensure that the value of :attr:`UnicodeEncodeError.start ` -retrieved by :c:func:`PyUnicodeEncodeError_GetStart` lie in +retrieved by :c:func:`PyUnicodeEncodeError_GetStart` lies in ``[0, max(0, objlen - 1)]`` where *objlen* is the length of :attr:`UnicodeEncodeError.object `. Similar arguments apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 124b591ee3a13f..287cbc25305964 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2712,7 +2712,7 @@ set_unicodefromstring(PyObject **attr, const char *value) * Adjust the (inclusive) 'start' value of a UnicodeError object. * * The 'start' can be negative or not, but when adjusting the value, - * we clip it in [0, max(0, objlen - 1)] but do not intepret it as + * we clip it in [0, max(0, objlen - 1)] and do not interpret it as * a relative offset. */ static inline Py_ssize_t @@ -2732,8 +2732,8 @@ unicode_error_adjust_start(Py_ssize_t start, Py_ssize_t objlen) * Adjust the (exclusive) 'end' value of a UnicodeError object. * * The 'end' can be negative or not, but when adjusting the value, - * we clip it in [min(1, objlen), max(min(1, objlen), objlen)] but - * do not intepret it as a relative offset. + * we clip it in [min(1, objlen), max(min(1, objlen), objlen)] and + * do not interpret it as a relative offset. */ static inline Py_ssize_t unicode_error_adjust_end(Py_ssize_t end, Py_ssize_t objlen) From 3b78409878c39d5afa344f7284b57104f7e765c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 8 Dec 2024 18:31:10 +0100 Subject: [PATCH 137/427] gh-87138: convert SHA-3 object type to heap type (GH-127670) --- Modules/sha3module.c | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/Modules/sha3module.c b/Modules/sha3module.c index ca839dc55e0519..b13e6a9de10114 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -71,13 +71,13 @@ typedef struct { static SHA3object * newSHA3object(PyTypeObject *type) { - SHA3object *newobj; - newobj = (SHA3object *)PyObject_New(SHA3object, type); + SHA3object *newobj = PyObject_GC_New(SHA3object, type); if (newobj == NULL) { return NULL; } HASHLIB_INIT_MUTEX(newobj); + PyObject_GC_Track(newobj); return newobj; } @@ -166,15 +166,32 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) /* Internal methods for a hash object */ +static int +SHA3_clear(SHA3object *self) +{ + if (self->hash_state != NULL) { + Hacl_Hash_SHA3_free(self->hash_state); + self->hash_state = NULL; + } + return 0; +} + static void SHA3_dealloc(SHA3object *self) { - Hacl_Hash_SHA3_free(self->hash_state); PyTypeObject *tp = Py_TYPE(self); - PyObject_Free(self); + PyObject_GC_UnTrack(self); + (void)SHA3_clear(self); + tp->tp_free(self); Py_DECREF(tp); } +static int +SHA3_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} /* External methods for a hash object */ @@ -335,6 +352,7 @@ static PyObject * SHA3_get_capacity_bits(SHA3object *self, void *closure) { uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state) * 8; + assert(rate <= 1600); int capacity = 1600 - rate; return PyLong_FromLong(capacity); } @@ -366,12 +384,14 @@ static PyGetSetDef SHA3_getseters[] = { #define SHA3_TYPE_SLOTS(type_slots_obj, type_doc, type_methods, type_getseters) \ static PyType_Slot type_slots_obj[] = { \ + {Py_tp_clear, SHA3_clear}, \ {Py_tp_dealloc, SHA3_dealloc}, \ + {Py_tp_traverse, SHA3_traverse}, \ {Py_tp_doc, (char*)type_doc}, \ {Py_tp_methods, type_methods}, \ {Py_tp_getset, type_getseters}, \ {Py_tp_new, py_sha3_new}, \ - {0,0} \ + {0, NULL} \ } // Using _PyType_GetModuleState() on these types is safe since they @@ -380,7 +400,8 @@ static PyGetSetDef SHA3_getseters[] = { static PyType_Spec type_spec_obj = { \ .name = "_sha3." type_name, \ .basicsize = sizeof(SHA3object), \ - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, \ + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE \ + | Py_TPFLAGS_HAVE_GC, \ .slots = type_slots \ } @@ -444,9 +465,7 @@ _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) result = PyBytes_FromStringAndSize((const char *)digest, digestlen); } - if (digest != NULL) { - PyMem_Free(digest); - } + PyMem_Free(digest); return result; } @@ -563,7 +582,7 @@ _sha3_clear(PyObject *module) static void _sha3_free(void *module) { - _sha3_clear((PyObject *)module); + (void)_sha3_clear((PyObject *)module); } static int From 2367759212f609b8ddf3218003b3ccd8e72849ae Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 8 Dec 2024 18:01:55 +0000 Subject: [PATCH 138/427] [doc] Fix typos in `interpreter_definition.md` (#127742) --- Tools/cases_generator/interpreter_definition.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index 203286834e3e3f..d50c420307852f 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -309,7 +309,7 @@ This might become (if it was an instruction): ### More examples -For explanations see "Generating the interpreter" below.) +For explanations see "Generating the interpreter" below. ```C op ( CHECK_HAS_INSTANCE_VALUES, (owner -- owner) ) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); @@ -371,7 +371,7 @@ For explanations see "Generating the interpreter" below.) A _family_ maps a specializable instruction to its specializations. -Example: These opcodes all share the same instruction format): +Example: These opcodes all share the same instruction format: ```C family(load_attr) = { LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_SLOT }; ``` @@ -393,7 +393,7 @@ which can be easily inserted. What is more complex is ensuring the correct stack and not generating excess pops and pushes. For example, in `CHECK_HAS_INSTANCE_VALUES`, `owner` occurs in the input, so it cannot be -redefined. Thus it doesn't need to written and can be read without adjusting the stack pointer. +redefined. Thus it doesn't need to be written and can be read without adjusting the stack pointer. The C code generated for `CHECK_HAS_INSTANCE_VALUES` would look something like: ```C From 7f8ec523021427a5c1ab3ce0cdd6e4bb909f1dc5 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 8 Dec 2024 18:45:09 +0000 Subject: [PATCH 139/427] GH-127381: pathlib ABCs: remove `PathBase.unlink()` and `rmdir()` (#127736) Virtual filesystems don't always make a distinction between deleting files and empty directories, and sometimes support deleting non-empty directories in a single operation. Here we remove `PathBase.unlink()` and `rmdir()`, leaving `_delete()` as the sole deletion method, now made abstract. I hope to drop the underscore prefix later on. --- Lib/pathlib/_abc.py | 43 +++-------------- Lib/pathlib/_local.py | 16 +++++-- Lib/test/test_pathlib/test_pathlib.py | 19 ++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 56 +++++------------------ 4 files changed, 48 insertions(+), 86 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 820970fcd5889b..309eab2ff855c3 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -840,6 +840,12 @@ def copy_into(self, target_dir, *, follow_symlinks=True, dirs_exist_ok=dirs_exist_ok, preserve_metadata=preserve_metadata) + def _delete(self): + """ + Delete this file or directory (including all sub-directories). + """ + raise UnsupportedOperation(self._unsupported_msg('_delete()')) + def move(self, target): """ Recursively move this file or directory tree to the given destination. @@ -874,43 +880,6 @@ def lchmod(self, mode): """ self.chmod(mode, follow_symlinks=False) - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - raise UnsupportedOperation(self._unsupported_msg('unlink()')) - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - raise UnsupportedOperation(self._unsupported_msg('rmdir()')) - - def _delete(self): - """ - Delete this file or directory (including all sub-directories). - """ - if self.is_symlink() or self.is_junction(): - self.unlink() - elif self.is_dir(): - self._rmtree() - else: - self.unlink() - - def _rmtree(self): - def on_error(err): - raise err - results = self.walk( - on_error=on_error, - top_down=False, # So we rmdir() empty directories. - follow_symlinks=False) - for dirpath, _, filenames in results: - for filename in filenames: - filepath = dirpath / filename - filepath.unlink() - dirpath.rmdir() - def owner(self, *, follow_symlinks=True): """ Return the login name of the file owner. diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 250bc12956f5bc..f87069ce70a2de 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -846,10 +846,18 @@ def rmdir(self): """ os.rmdir(self) - def _rmtree(self): - # Lazy import to improve module import time - import shutil - shutil.rmtree(self) + def _delete(self): + """ + Delete this file or directory (including all sub-directories). + """ + if self.is_symlink() or self.is_junction(): + self.unlink() + elif self.is_dir(): + # Lazy import to improve module import time + import shutil + shutil.rmtree(self) + else: + self.unlink() def rename(self, target): """ diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 8c9049f15d5bf9..ce0f4748c860b1 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1352,6 +1352,25 @@ def test_group_no_follow_symlinks(self): self.assertEqual(expected_gid, gid_2) self.assertEqual(expected_name, link.group(follow_symlinks=False)) + def test_unlink(self): + p = self.cls(self.base) / 'fileA' + p.unlink() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + def test_unlink_missing_ok(self): + p = self.cls(self.base) / 'fileAAA' + self.assertFileNotFound(p.unlink) + p.unlink(missing_ok=True) + + def test_rmdir(self): + p = self.cls(self.base) / 'dirA' + for q in p.iterdir(): + q.unlink() + p.rmdir() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + @needs_symlinks def test_delete_symlink(self): tmp = self.cls(self.base, 'delete') diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index bf9ae6cc8a2433..675abf30a9f13c 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1370,8 +1370,6 @@ def test_unsupported_operation(self): self.assertRaises(e, p.touch) self.assertRaises(e, p.chmod, 0o755) self.assertRaises(e, p.lchmod, 0o755) - self.assertRaises(e, p.unlink) - self.assertRaises(e, p.rmdir) self.assertRaises(e, p.owner) self.assertRaises(e, p.group) self.assertRaises(e, p.as_uri) @@ -1493,31 +1491,18 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): self.parent.mkdir(parents=True, exist_ok=True) self.mkdir(mode, parents=False, exist_ok=exist_ok) - def unlink(self, missing_ok=False): - path = str(self) - name = self.name - parent = str(self.parent) - if path in self._directories: - raise IsADirectoryError(errno.EISDIR, "Is a directory", path) - elif path in self._files: - self._directories[parent].remove(name) - del self._files[path] - elif not missing_ok: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - - def rmdir(self): + def _delete(self): path = str(self) if path in self._files: - raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) - elif path not in self._directories: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - elif self._directories[path]: - raise OSError(errno.ENOTEMPTY, "Directory not empty", path) - else: - name = self.name - parent = str(self.parent) - self._directories[parent].remove(name) + del self._files[path] + elif path in self._directories: + for name in list(self._directories[path]): + self.joinpath(name)._delete() del self._directories[path] + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + parent = str(self.parent) + self._directories[parent].remove(self.name) class DummyPathTest(DummyPurePathTest): @@ -2245,30 +2230,11 @@ def test_is_char_device_false(self): self.assertIs((P / 'fileA\udfff').is_char_device(), False) self.assertIs((P / 'fileA\x00').is_char_device(), False) - def test_unlink(self): - p = self.cls(self.base) / 'fileA' - p.unlink() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) - - def test_unlink_missing_ok(self): - p = self.cls(self.base) / 'fileAAA' - self.assertFileNotFound(p.unlink) - p.unlink(missing_ok=True) - - def test_rmdir(self): - p = self.cls(self.base) / 'dirA' - for q in p.iterdir(): - q.unlink() - p.rmdir() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) - def test_delete_file(self): p = self.cls(self.base) / 'fileA' p._delete() self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) + self.assertFileNotFound(p._delete) def test_delete_dir(self): base = self.cls(self.base) @@ -2347,7 +2313,7 @@ def setUp(self): def tearDown(self): base = self.cls(self.base) - base._rmtree() + base._delete() def test_walk_topdown(self): walker = self.walk_path.walk() From a03efb533a58fd13fb0cc7f4a5c02c8406a407bd Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Sun, 8 Dec 2024 10:46:34 -0800 Subject: [PATCH 140/427] gh-127734: improve signature of `urllib.request.HTTPPasswordMgrWithPriorAuth.__init__` (#127735) improve signature of urllib.request.HTTPPasswordMgrWithPriorAuth.__init__ --- Lib/urllib/request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 7ef85431b718ad..c5a6a18a32bba1 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -876,9 +876,9 @@ def find_user_password(self, realm, authuri): class HTTPPasswordMgrWithPriorAuth(HTTPPasswordMgrWithDefaultRealm): - def __init__(self, *args, **kwargs): + def __init__(self): self.authenticated = {} - super().__init__(*args, **kwargs) + super().__init__() def add_password(self, realm, uri, user, passwd, is_authenticated=False): self.update_authenticated(uri, is_authenticated) From be07edf511365ce554c0535b535bb5726266a17a Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 9 Dec 2024 02:34:28 +0100 Subject: [PATCH 141/427] gh-127111: Emscripten Move link flags from `LDFLAGS_NODIST` to `LINKFORSHARED` (#127666) Corrects the usage of linking flags to avoid compilation errors related to the use of `-sEXPORTED_FUNCTIONS` when linking shared libraries. --- configure | 10 +++++----- configure.ac | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/configure b/configure index bcbab8dfcff190..57be576e3cae99 100755 --- a/configure +++ b/configure @@ -9430,14 +9430,14 @@ else $as_nop wasm_debug=no fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520" + as_fn_append LINKFORSHARED " -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520" as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" - as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV" - as_fn_append LDFLAGS_NODIST " -sEXPORTED_FUNCTIONS=_main,_Py_Version" - as_fn_append LDFLAGS_NODIST " -sSTACK_SIZE=5MB" + as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" + as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV" + as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version" + as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" if test "x$enable_wasm_dynamic_linking" = xyes then : diff --git a/configure.ac b/configure.ac index 922a125ea9608e..bd0221481c5341 100644 --- a/configure.ac +++ b/configure.ac @@ -2325,16 +2325,16 @@ AS_CASE([$ac_sys_system], AS_VAR_IF([Py_DEBUG], [yes], [wasm_debug=yes], [wasm_debug=no]) dnl Start with 20 MB and allow to grow - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"]) dnl map int64_t and uint64_t to JS bigint AS_VAR_APPEND([LDFLAGS_NODIST], [" -sWASM_BIGINT"]) dnl Include file system support - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV"]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version"]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sSTACK_SIZE=5MB"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) From 5876063d06ec55b10793f34bfe516c10f608665c Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 9 Dec 2024 03:01:37 +0100 Subject: [PATCH 142/427] gh-127503 Don't propagate native PATH to Emscripten Python (#127633) Modifies the handling of PATH to ensure that native executables aren't picked up when running under node. --- Tools/wasm/emscripten/node_entry.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs index 40ab1515cf28c1..98b8f572a7e762 100644 --- a/Tools/wasm/emscripten/node_entry.mjs +++ b/Tools/wasm/emscripten/node_entry.mjs @@ -35,11 +35,12 @@ const settings = { mountDirectories(Module); Module.FS.chdir(process.cwd()); Object.assign(Module.ENV, process.env); + delete Module.ENV.PATH; }, // Ensure that sys.executable, sys._base_executable, etc point to python.sh // not to this file. To properly handle symlinks, python.sh needs to compute // its own path. - thisProgram: process.argv[thisProgramIndex], + thisProgram: process.argv[thisProgramIndex].slice(thisProgram.length), // After python.sh come the arguments thatthe user passed to python.sh. arguments: process.argv.slice(thisProgramIndex + 1), }; From d8d12b37b5e5acb354db84b07dab8de64a6b9475 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 9 Dec 2024 03:03:11 +0100 Subject: [PATCH 143/427] gh-127503: Fix realpath handling in emscripten cli (#127632) Corrects the handling of realpath on Linux. --- Tools/wasm/emscripten/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index c998ed71309dad..6843b6fdeceb8c 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -223,7 +223,7 @@ def configure_emscripten_python(context, working_dir): if which grealpath > /dev/null; then # It has brew installed gnu core utils, use that REALPATH="grealpath -s" - elif which realpath > /dev/null && realpath --version 2&>1 | grep GNU > /dev/null; then + elif which realpath > /dev/null && realpath --version > /dev/null 2> /dev/null && realpath --version | grep GNU > /dev/null; then # realpath points to GNU realpath so use it. REALPATH="realpath -s" else From 2041a95e68ebf6d13f867e214ada28affa830669 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Mon, 9 Dec 2024 13:28:57 +0800 Subject: [PATCH 144/427] gh-126925: Modify how iOS test results are gathered (#127592) Adds a `use_system_log` config item to enable stdout/stderr redirection for Apple platforms. This log streaming is then used by a new iOS test runner script, allowing the display of test suite output at runtime. The iOS test runner script can be used by any Python project, not just the CPython test suite. --- Doc/c-api/init_config.rst | 9 + Doc/using/ios.rst | 53 ++- Doc/whatsnew/3.14.rst | 7 + Include/cpython/initconfig.h | 3 + Lib/_apple_support.py | 66 ++++ Lib/test/test_apple.py | 155 ++++++++ Lib/test/test_capi/test_config.py | 4 + Lib/test/test_embed.py | 2 + Makefile.pre.in | 26 +- ...-12-04-15-04-12.gh-issue-126821.lKCLVV.rst | 2 + ...-12-04-15-03-24.gh-issue-126925.uxAMK-.rst | 2 + Python/initconfig.c | 15 + Python/pylifecycle.c | 82 ++++ Python/stdlib_module_names.h | 1 + iOS/README.rst | 54 ++- iOS/testbed/__main__.py | 365 ++++++++++++++++++ .../iOSTestbed.xcodeproj/project.pbxproj | 2 + iOS/testbed/iOSTestbedTests/iOSTestbedTests.m | 2 + 18 files changed, 792 insertions(+), 58 deletions(-) create mode 100644 Lib/_apple_support.py create mode 100644 Lib/test/test_apple.py create mode 100644 Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst create mode 100644 Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst create mode 100644 iOS/testbed/__main__.py diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index d6569ddcf586fa..7497bf241fb10e 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1281,6 +1281,15 @@ PyConfig Default: ``1`` in Python config and ``0`` in isolated config. + .. c:member:: int use_system_logger + + If non-zero, ``stdout`` and ``stderr`` will be redirected to the system + log. + + Only available on macOS 10.12 and later, and on iOS. + + Default: ``0`` (don't use system log). + .. c:member:: int user_site_directory If non-zero, add the user site directory to :data:`sys.path`. diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index 4d4eb2031ee980..aa43f75ec35a6c 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -292,10 +292,12 @@ To add Python to an iOS Xcode project: 10. Add Objective C code to initialize and use a Python interpreter in embedded mode. You should ensure that: - * :c:member:`UTF-8 mode ` is *enabled*; - * :c:member:`Buffered stdio ` is *disabled*; - * :c:member:`Writing bytecode ` is *disabled*; - * :c:member:`Signal handlers ` are *enabled*; + * UTF-8 mode (:c:member:`PyPreConfig.utf8_mode`) is *enabled*; + * Buffered stdio (:c:member:`PyConfig.buffered_stdio`) is *disabled*; + * Writing bytecode (:c:member:`PyConfig.write_bytecode`) is *disabled*; + * Signal handlers (:c:member:`PyConfig.install_signal_handlers`) are *enabled*; + * System logging (:c:member:`PyConfig.use_system_logger`) is *enabled* + (optional, but strongly recommended); * ``PYTHONHOME`` for the interpreter is configured to point at the ``python`` subfolder of your app's bundle; and * The ``PYTHONPATH`` for the interpreter includes: @@ -324,6 +326,49 @@ modules in your app, some additional steps will be required: * If you're using a separate folder for third-party packages, ensure that folder is included as part of the ``PYTHONPATH`` configuration in step 10. +Testing a Python package +------------------------ + +The CPython source tree contains :source:`a testbed project ` that +is used to run the CPython test suite on the iOS simulator. This testbed can also +be used as a testbed project for running your Python library's test suite on iOS. + +After building or obtaining an iOS XCFramework (See :source:`iOS/README.rst` +for details), create a clone of the Python iOS testbed project by running: + +.. code-block:: bash + + $ python iOS/testbed clone --framework --app --app app-testbed + +You will need to modify the ``iOS/testbed`` reference to point to that +directory in the CPython source tree; any folders specified with the ``--app`` +flag will be copied into the cloned testbed project. The resulting testbed will +be created in the ``app-testbed`` folder. In this example, the ``module1`` and +``module2`` would be importable modules at runtime. If your project has +additional dependencies, they can be installed into the +``app-testbed/iOSTestbed/app_packages`` folder (using ``pip install --target +app-testbed/iOSTestbed/app_packages`` or similar). + +You can then use the ``app-testbed`` folder to run the test suite for your app, +For example, if ``module1.tests`` was the entry point to your test suite, you +could run: + +.. code-block:: bash + + $ python app-testbed run -- module1.tests + +This is the equivalent of running ``python -m module1.tests`` on a desktop +Python build. Any arguments after the ``--`` will be passed to the testbed as +if they were arguments to ``python -m`` on a desktop machine. + +You can also open the testbed project in Xcode by running: + +.. code-block:: bash + + $ open app-testbed/iOSTestbed.xcodeproj + +This will allow you to use the full Xcode suite of tools for debugging. + App Store Compliance ==================== diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b300e348679438..b71d31f9742fe0 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -245,6 +245,13 @@ Other language changes making it a :term:`generic type`. (Contributed by Brian Schubert in :gh:`126012`.) +* iOS and macOS apps can now be configured to redirect ``stdout`` and + ``stderr`` content to the system log. (Contributed by Russell Keith-Magee in + :gh:`127592`.) + +* The iOS testbed is now able to stream test output while the test is running. + The testbed can also be used to run the test suite of projects other than + CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.) New modules =========== diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index f69c586a4f96f3..8ef19f677066c2 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -179,6 +179,9 @@ typedef struct PyConfig { int use_frozen_modules; int safe_path; int int_max_str_digits; +#ifdef __APPLE__ + int use_system_logger; +#endif int cpu_count; #ifdef Py_GIL_DISABLED diff --git a/Lib/_apple_support.py b/Lib/_apple_support.py new file mode 100644 index 00000000000000..92febdcf587070 --- /dev/null +++ b/Lib/_apple_support.py @@ -0,0 +1,66 @@ +import io +import sys + + +def init_streams(log_write, stdout_level, stderr_level): + # Redirect stdout and stderr to the Apple system log. This method is + # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger + # is enabled. + sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors) + sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors) + + +class SystemLog(io.TextIOWrapper): + def __init__(self, log_write, level, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(LogStream(log_write, level), **kwargs) + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line, so split + # the string before sending it to the superclass. + for line in s.splitlines(keepends=True): + super().write(line) + + return len(s) + + +class LogStream(io.RawIOBase): + def __init__(self, log_write, level): + self.log_write = log_write + self.level = level + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/test/test_apple.py b/Lib/test/test_apple.py new file mode 100644 index 00000000000000..ab5296afad1d3f --- /dev/null +++ b/Lib/test/test_apple.py @@ -0,0 +1,155 @@ +import unittest +from _apple_support import SystemLog +from test.support import is_apple +from unittest.mock import Mock, call + +if not is_apple: + raise unittest.SkipTest("Apple-specific") + + +# Test redirection of stdout and stderr to the Apple system log. +class TestAppleSystemLogOutput(unittest.TestCase): + maxDiff = None + + def assert_writes(self, output): + self.assertEqual( + self.log_write.mock_calls, + [ + call(self.log_level, line) + for line in output + ] + ) + + self.log_write.reset_mock() + + def setUp(self): + self.log_write = Mock() + self.log_level = 42 + self.log = SystemLog(self.log_write, self.log_level, errors="replace") + + def test_repr(self): + self.assertEqual(repr(self.log), "") + self.assertEqual(repr(self.log.buffer), "") + + def test_log_config(self): + self.assertIs(self.log.writable(), True) + self.assertIs(self.log.readable(), False) + + self.assertEqual("UTF-8", self.log.encoding) + self.assertEqual("replace", self.log.errors) + + self.assertIs(self.log.line_buffering, True) + self.assertIs(self.log.write_through, False) + + def test_empty_str(self): + self.log.write("") + self.log.flush() + + self.assert_writes([]) + + def test_simple_str(self): + self.log.write("hello world\n") + + self.assert_writes([b"hello world\n"]) + + def test_buffered_str(self): + self.log.write("h") + self.log.write("ello") + self.log.write(" ") + self.log.write("world\n") + self.log.write("goodbye.") + self.log.flush() + + self.assert_writes([b"hello world\n", b"goodbye."]) + + def test_manual_flush(self): + self.log.write("Hello") + + self.assert_writes([]) + + self.log.write(" world\nHere for a while...\nGoodbye") + self.assert_writes([b"Hello world\n", b"Here for a while...\n"]) + + self.log.write(" world\nHello again") + self.assert_writes([b"Goodbye world\n"]) + + self.log.flush() + self.assert_writes([b"Hello again"]) + + def test_non_ascii(self): + # Spanish + self.log.write("ol\u00e9\n") + self.assert_writes([b"ol\xc3\xa9\n"]) + + # Chinese + self.log.write("\u4e2d\u6587\n") + self.assert_writes([b"\xe4\xb8\xad\xe6\x96\x87\n"]) + + # Printing Non-BMP emoji + self.log.write("\U0001f600\n") + self.assert_writes([b"\xf0\x9f\x98\x80\n"]) + + # Non-encodable surrogates are replaced + self.log.write("\ud800\udc00\n") + self.assert_writes([b"??\n"]) + + def test_modified_null(self): + # Null characters are logged using "modified UTF-8". + self.log.write("\u0000\n") + self.assert_writes([b"\xc0\x80\n"]) + self.log.write("a\u0000\n") + self.assert_writes([b"a\xc0\x80\n"]) + self.log.write("\u0000b\n") + self.assert_writes([b"\xc0\x80b\n"]) + self.log.write("a\u0000b\n") + self.assert_writes([b"a\xc0\x80b\n"]) + + def test_nonstandard_str(self): + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + self.log.write(CustomStr("custom\n")) + self.assert_writes([b"custom\n"]) + + def test_non_str(self): + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + self.log.write(obj) + + def test_byteslike_in_buffer(self): + # The underlying buffer *can* accept bytes-like objects + self.log.buffer.write(bytearray(b"hello")) + self.log.flush() + + self.log.buffer.write(b"") + self.log.flush() + + self.log.buffer.write(b"goodbye") + self.log.flush() + + self.assert_writes([b"hello", b"goodbye"]) + + def test_non_byteslike_in_buffer(self): + for obj in ["hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + self.log.buffer.write(obj) diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 77730ad2f32085..a3179efe4a8235 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -110,6 +110,10 @@ def test_config_get(self): options.extend(( ("_pystats", bool, None), )) + if support.is_apple: + options.extend(( + ("use_system_logger", bool, None), + )) for name, option_type, sys_attr in options: with self.subTest(name=name, option_type=option_type, diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 5c38b28322deb4..7110fb889f3c8e 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -649,6 +649,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT.update({ 'legacy_windows_stdio': False, }) + if support.is_apple: + CONFIG_COMPAT['use_system_logger'] = False CONFIG_PYTHON = dict(CONFIG_COMPAT, _config_init=API_PYTHON, diff --git a/Makefile.pre.in b/Makefile.pre.in index dd8a3ab82eacd2..7b66802147dc3a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2146,7 +2146,6 @@ testuniversal: all # This must be run *after* a `make install` has completed the build. The # `--with-framework-name` argument *cannot* be used when configuring the build. XCFOLDER:=iOSTestbed.$(MULTIARCH).$(shell date +%s) -XCRESULT=$(XCFOLDER)/$(MULTIARCH).xcresult .PHONY: testios testios: @if test "$(MACHDEP)" != "ios"; then \ @@ -2165,29 +2164,12 @@ testios: echo "Cannot find a finalized iOS Python.framework. Have you run 'make install' to finalize the framework build?"; \ exit 1;\ fi - # Copy the testbed project into the build folder - cp -r $(srcdir)/iOS/testbed $(XCFOLDER) - # Copy the framework from the install location to the testbed project. - cp -r $(PYTHONFRAMEWORKPREFIX)/* $(XCFOLDER)/Python.xcframework/ios-arm64_x86_64-simulator - - # Run the test suite for the Xcode project, targeting the iOS simulator. - # If the suite fails, touch a file in the test folder as a marker - if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) -derivedDataPath $(XCFOLDER)/DerivedData ; then \ - touch $(XCFOLDER)/failed; \ - fi - # Regardless of success or failure, extract and print the test output - xcrun xcresulttool get --path $(XCRESULT) \ - --id $$( \ - xcrun xcresulttool get --path $(XCRESULT) --format json | \ - $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['actions']['_values'][0]['actionResult']['logRef']['id']['_value'])" \ - ) \ - --format json | \ - $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['subsections']['_values'][1]['subsections']['_values'][0]['emittedOutput']['_value'])" + # Clone the testbed project into the XCFOLDER + $(PYTHON_FOR_BUILD) $(srcdir)/iOS/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" - @if test -e $(XCFOLDER)/failed ; then \ - exit 1; \ - fi + # Run the testbed project + $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run -- test -uall --single-process --rerun -W # Like test, but using --slow-ci which enables all test resources and use # longer timeout. Run an optional pybuildbot.identify script to include diff --git a/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst b/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst new file mode 100644 index 00000000000000..677acf5baab3fa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst @@ -0,0 +1,2 @@ +macOS and iOS apps can now choose to redirect stdout and stderr to the +system log during interpreter configuration. diff --git a/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst b/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst new file mode 100644 index 00000000000000..fb307c7cb9bf1d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst @@ -0,0 +1,2 @@ +iOS test results are now streamed during test execution, and the deprecated +xcresulttool is no longer used. diff --git a/Python/initconfig.c b/Python/initconfig.c index 438f8a5c1cf1ce..7851b86db1f6d0 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -168,6 +168,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS), SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS), SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS), +#ifdef __APPLE__ + SPEC(use_system_logger, BOOL, PUBLIC, NO_SYS), +#endif SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_user_site SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS), @@ -884,6 +887,9 @@ config_check_consistency(const PyConfig *config) assert(config->cpu_count != 0); // config->use_frozen_modules is initialized later // by _PyConfig_InitImportConfig(). +#ifdef __APPLE__ + assert(config->use_system_logger >= 0); +#endif #ifdef Py_STATS assert(config->_pystats >= 0); #endif @@ -986,6 +992,9 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->_is_python_build = 0; config->code_debug_ranges = 1; config->cpu_count = -1; +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif #ifdef Py_GIL_DISABLED config->enable_gil = _PyConfig_GIL_DEFAULT; config->tlbc_enabled = 1; @@ -1015,6 +1024,9 @@ config_init_defaults(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif } @@ -1049,6 +1061,9 @@ PyConfig_InitIsolatedConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ceb30e9f02df2c..06418123d6dd9b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -45,7 +45,9 @@ #endif #if defined(__APPLE__) +# include # include +# include #endif #ifdef HAVE_SIGNAL_H @@ -75,6 +77,9 @@ static PyStatus init_sys_streams(PyThreadState *tstate); #ifdef __ANDROID__ static PyStatus init_android_streams(PyThreadState *tstate); #endif +#if defined(__APPLE__) +static PyStatus init_apple_streams(PyThreadState *tstate); +#endif static void wait_for_thread_shutdown(PyThreadState *tstate); static void finalize_subinterpreters(void); static void call_ll_exitfuncs(_PyRuntimeState *runtime); @@ -1257,6 +1262,14 @@ init_interp_main(PyThreadState *tstate) return status; } #endif +#if defined(__APPLE__) + if (config->use_system_logger) { + status = init_apple_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } +#endif #ifdef Py_DEBUG run_presite(tstate); @@ -2933,6 +2946,75 @@ init_android_streams(PyThreadState *tstate) #endif // __ANDROID__ +#if defined(__APPLE__) + +static PyObject * +apple_log_write_impl(PyObject *self, PyObject *args) +{ + int logtype = 0; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "iy", &logtype, &text)) { + return NULL; + } + + // Call the underlying Apple logging API. The os_log unified logging APIs + // were introduced in macOS 10.12, iOS 10.0, tvOS 10.0, and watchOS 3.0; + // this call is a no-op on older versions. + #if TARGET_OS_IPHONE || (TARGET_OS_OSX && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12) + // Pass the user-provided text through explicit %s formatting + // to avoid % literals being interpreted as a formatting directive. + os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); + #endif + Py_RETURN_NONE; +} + + +static PyMethodDef apple_log_write_method = { + "apple_log_write", apple_log_write_impl, METH_VARARGS +}; + + +static PyStatus +init_apple_streams(PyThreadState *tstate) +{ + PyStatus status = _PyStatus_OK(); + PyObject *_apple_support = NULL; + PyObject *apple_log_write = NULL; + PyObject *result = NULL; + + _apple_support = PyImport_ImportModule("_apple_support"); + if (_apple_support == NULL) { + goto error; + } + + apple_log_write = PyCFunction_New(&apple_log_write_method, NULL); + if (apple_log_write == NULL) { + goto error; + } + + // Initialize the logging streams, sending stdout -> Default; stderr -> Error + result = PyObject_CallMethod( + _apple_support, "init_streams", "Oii", + apple_log_write, OS_LOG_TYPE_DEFAULT, OS_LOG_TYPE_ERROR); + if (result == NULL) { + goto error; + } + + goto done; + +error: + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Apple log streams"); + +done: + Py_XDECREF(result); + Py_XDECREF(apple_log_write); + Py_XDECREF(_apple_support); + return status; +} + +#endif // __APPLE__ + static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index c8cdb933bb108f..584b050fc4bb6e 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -6,6 +6,7 @@ static const char* _Py_stdlib_module_names[] = { "_abc", "_aix_support", "_android_support", +"_apple_support", "_ast", "_asyncio", "_bisect", diff --git a/iOS/README.rst b/iOS/README.rst index e33455eef8f44a..9cea98cf1abbfa 100644 --- a/iOS/README.rst +++ b/iOS/README.rst @@ -285,52 +285,42 @@ This will: * Install the Python iOS framework into the copy of the testbed project; and * Run the test suite on an "iPhone SE (3rd generation)" simulator. -While the test suite is running, Xcode does not display any console output. -After showing some Xcode build commands, the console output will print ``Testing -started``, and then appear to stop. It will remain in this state until the test -suite completes. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 -minutes to run; a couple of extra minutes is required to boot and prepare the -iOS simulator. - On success, the test suite will exit and report successful completion of the -test suite. No output of the Python test suite will be displayed. - -On failure, the output of the Python test suite *will* be displayed. This will -show the details of the tests that failed. +test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 +minutes to run; a couple of extra minutes is required to compile the testbed +project, and then boot and prepare the iOS simulator. Debugging test failures ----------------------- -The easiest way to diagnose a single test failure is to open the testbed project -in Xcode and run the tests from there using the "Product > Test" menu item. - -To test in Xcode, you must ensure the testbed project has a copy of a compiled -framework. If you've configured your build with the default install location of -``iOS/Frameworks``, you can copy from that location into the test project. To -test on an ARM64 simulator, run:: - - $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/* - $ cp -r iOS/Frameworks/arm64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator +Running ``make test`` generates a standalone version of the ``iOS/testbed`` +project, and runs the full test suite. It does this using ``iOS/testbed`` +itself - the folder is an executable module that can be used to create and run +a clone of the testbed project. -To test on an x86-64 simulator, run:: +You can generate your own standalone testbed instance by running:: - $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/* - $ cp -r iOS/Frameworks/x86_64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator + $ python iOS/testbed clone --framework iOS/Frameworks/arm64-iphonesimulator my-testbed -To test on a physical device:: +This invocation assumes that ``iOS/Frameworks/arm64-iphonesimulator`` is the +path to the iOS simulator framework for your platform (ARM64 in this case); +``my-testbed`` is the name of the folder for the new testbed clone. - $ rm -rf iOS/testbed/Python.xcframework/ios-arm64/* - $ cp -r iOS/Frameworks/arm64-iphoneos/* iOS/testbed/Python.xcframework/ios-arm64 +You can then use the ``my-testbed`` folder to run the Python test suite, +passing in any command line arguments you may require. For example, if you're +trying to diagnose a failure in the ``os`` module, you might run:: -Alternatively, you can configure your build to install directly into the -testbed project. For a simulator, use:: + $ python my-testbed run -- test -W test_os - --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator +This is the equivalent of running ``python -m test -W test_os`` on a desktop +Python build. Any arguments after the ``--`` will be passed to testbed as if +they were arguments to ``python -m`` on a desktop machine. -For a physical device, use:: +You can also open the testbed project in Xcode by running:: - --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64 + $ open my-testbed/iOSTestbed.xcodeproj +This will allow you to use the full Xcode suite of tools for debugging. Testing on an iOS device ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py new file mode 100644 index 00000000000000..22570ee0f3ed04 --- /dev/null +++ b/iOS/testbed/__main__.py @@ -0,0 +1,365 @@ +import argparse +import asyncio +import json +import plistlib +import shutil +import subprocess +import sys +from contextlib import asynccontextmanager +from datetime import datetime +from pathlib import Path + + +DECODE_ARGS = ("UTF-8", "backslashreplace") + + +# Work around a bug involving sys.exit and TaskGroups +# (https://github.com/python/cpython/issues/101515). +def exit(*args): + raise MySystemExit(*args) + + +class MySystemExit(Exception): + pass + + +# All subprocesses are executed through this context manager so that no matter +# what happens, they can always be cancelled from another task, and they will +# always be cleaned up on exit. +@asynccontextmanager +async def async_process(*args, **kwargs): + process = await asyncio.create_subprocess_exec(*args, **kwargs) + try: + yield process + finally: + if process.returncode is None: + # Allow a reasonably long time for Xcode to clean itself up, + # because we don't want stale emulators left behind. + timeout = 10 + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout) + except TimeoutError: + print( + f"Command {args} did not terminate after {timeout} seconds " + f" - sending SIGKILL" + ) + process.kill() + + # Even after killing the process we must still wait for it, + # otherwise we'll get the warning "Exception ignored in __del__". + await asyncio.wait_for(process.wait(), timeout=1) + + +async def async_check_output(*args, **kwargs): + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs + ) as process: + stdout, stderr = await process.communicate() + if process.returncode == 0: + return stdout.decode(*DECODE_ARGS) + else: + raise subprocess.CalledProcessError( + process.returncode, + args, + stdout.decode(*DECODE_ARGS), + stderr.decode(*DECODE_ARGS), + ) + + +# Return a list of UDIDs associated with booted simulators +async def list_devices(): + # List the testing simulators, in JSON format + raw_json = await async_check_output( + "xcrun", "simctl", "--set", "testing", "list", "-j" + ) + json_data = json.loads(raw_json) + + # Filter out the booted iOS simulators + return [ + simulator["udid"] + for runtime, simulators in json_data["devices"].items() + for simulator in simulators + if runtime.split(".")[-1].startswith("iOS") and simulator["state"] == "Booted" + ] + + +async def find_device(initial_devices): + while True: + new_devices = set(await list_devices()).difference(initial_devices) + if len(new_devices) == 0: + await asyncio.sleep(1) + elif len(new_devices) == 1: + udid = new_devices.pop() + print(f"{datetime.now():%Y-%m-%d %H:%M:%S}: New test simulator detected") + print(f"UDID: {udid}") + return udid + else: + exit(f"Found more than one new device: {new_devices}") + + +async def log_stream_task(initial_devices): + # Wait up to 5 minutes for the build to complete and the simulator to boot. + udid = await asyncio.wait_for(find_device(initial_devices), 5 * 60) + + # Stream the iOS device's logs, filtering out messages that come from the + # XCTest test suite (catching NSLog messages from the test method), or + # Python itself (catching stdout/stderr content routed to the system log + # with config->use_system_logger). + args = [ + "xcrun", + "simctl", + "--set", + "testing", + "spawn", + udid, + "log", + "stream", + "--style", + "compact", + "--predicate", + ( + 'senderImagePath ENDSWITH "/iOSTestbedTests.xctest/iOSTestbedTests"' + ' OR senderImagePath ENDSWITH "/Python.framework/Python"' + ), + ] + + async with async_process( + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + suppress_dupes = False + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + # The iOS log streamer can sometimes lag; when it does, it outputs + # a warning about messages being dropped... often multiple times. + # Only print the first of these duplicated warnings. + if line.startswith("=== Messages dropped "): + if not suppress_dupes: + suppress_dupes = True + sys.stdout.write(line) + else: + suppress_dupes = False + sys.stdout.write(line) + + +async def xcode_test(location, simulator): + # Run the test suite on the named simulator + args = [ + "xcodebuild", + "test", + "-project", + str(location / "iOSTestbed.xcodeproj"), + "-scheme", + "iOSTestbed", + "-destination", + f"platform=iOS Simulator,name={simulator}", + "-resultBundlePath", + str(location / f"{datetime.now():%Y%m%d-%H%M%S}.xcresult"), + "-derivedDataPath", + str(location / "DerivedData"), + ] + async with async_process( + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + sys.stdout.write(line) + + status = await asyncio.wait_for(process.wait(), timeout=1) + exit(status) + + +def clone_testbed( + source: Path, + target: Path, + framework: Path, + apps: list[Path], +) -> None: + if target.exists(): + print(f"{target} already exists; aborting without creating project.") + sys.exit(10) + + if framework is None: + if not (source / "Python.xcframework/ios-arm64_x86_64-simulator/bin").is_dir(): + print( + f"The testbed being cloned ({source}) does not contain " + f"a simulator framework. Re-run with --framework" + ) + sys.exit(11) + else: + if not framework.is_dir(): + print(f"{framework} does not exist.") + sys.exit(12) + elif not ( + framework.suffix == ".xcframework" + or (framework / "Python.framework").is_dir() + ): + print( + f"{framework} is not an XCframework, " + f"or a simulator slice of a framework build." + ) + sys.exit(13) + + print("Cloning testbed project...") + shutil.copytree(source, target) + + if framework is not None: + if framework.suffix == ".xcframework": + print("Installing XCFramework...") + xc_framework_path = target / "Python.xcframework" + shutil.rmtree(xc_framework_path) + shutil.copytree(framework, xc_framework_path) + else: + print("Installing simulator Framework...") + sim_framework_path = ( + target / "Python.xcframework" / "ios-arm64_x86_64-simulator" + ) + shutil.rmtree(sim_framework_path) + shutil.copytree(framework, sim_framework_path) + else: + print("Using pre-existing iOS framework.") + + for app_src in apps: + print(f"Installing app {app_src.name!r}...") + app_target = target / f"iOSTestbed/app/{app_src.name}" + if app_target.is_dir(): + shutil.rmtree(app_target) + shutil.copytree(app_src, app_target) + + print(f"Testbed project created in {target}") + + +def update_plist(testbed_path, args): + # Add the test runner arguments to the testbed's Info.plist file. + info_plist = testbed_path / "iOSTestbed" / "iOSTestbed-Info.plist" + with info_plist.open("rb") as f: + info = plistlib.load(f) + + info["TestArgs"] = args + + with info_plist.open("wb") as f: + plistlib.dump(info, f) + + +async def run_testbed(simulator: str, args: list[str]): + location = Path(__file__).parent + print("Updating plist...") + update_plist(location, args) + + # Get the list of devices that are booted at the start of the test run. + # The simulator started by the test suite will be detected as the new + # entry that appears on the device list. + initial_devices = await list_devices() + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(log_stream_task(initial_devices)) + tg.create_task(xcode_test(location, simulator)) + except* MySystemExit as e: + raise SystemExit(*e.exceptions[0].args) from None + except* subprocess.CalledProcessError as e: + # Extract it from the ExceptionGroup so it can be handled by `main`. + raise e.exceptions[0] + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Manages the process of testing a Python project in the iOS simulator." + ), + ) + + subcommands = parser.add_subparsers(dest="subcommand") + + clone = subcommands.add_parser( + "clone", + description=( + "Clone the testbed project, copying in an iOS Python framework and" + "any specified application code." + ), + help="Clone a testbed project to a new location.", + ) + clone.add_argument( + "--framework", + help=( + "The location of the XCFramework (or simulator-only slice of an " + "XCFramework) to use when running the testbed" + ), + ) + clone.add_argument( + "--app", + dest="apps", + action="append", + default=[], + help="The location of any code to include in the testbed project", + ) + clone.add_argument( + "location", + help="The path where the testbed will be cloned.", + ) + + run = subcommands.add_parser( + "run", + usage="%(prog)s [-h] [--simulator SIMULATOR] -- [ ...]", + description=( + "Run a testbed project. The arguments provided after `--` will be " + "passed to the running iOS process as if they were arguments to " + "`python -m`." + ), + help="Run a testbed project", + ) + run.add_argument( + "--simulator", + default="iPhone SE (3rd Generation)", + help="The name of the simulator to use (default: 'iPhone SE (3rd Generation)')", + ) + + try: + pos = sys.argv.index("--") + testbed_args = sys.argv[1:pos] + test_args = sys.argv[pos + 1 :] + except ValueError: + testbed_args = sys.argv[1:] + test_args = [] + + context = parser.parse_args(testbed_args) + + if context.subcommand == "clone": + clone_testbed( + source=Path(__file__).parent, + target=Path(context.location), + framework=Path(context.framework) if context.framework else None, + apps=[Path(app) for app in context.apps], + ) + elif context.subcommand == "run": + if test_args: + if not ( + Path(__file__).parent / "Python.xcframework/ios-arm64_x86_64-simulator/bin" + ).is_dir(): + print( + f"Testbed does not contain a compiled iOS framework. Use " + f"`python {sys.argv[0]} clone ...` to create a runnable " + f"clone of this testbed." + ) + sys.exit(20) + + asyncio.run( + run_testbed( + simulator=context.simulator, + args=test_args, + ) + ) + else: + print(f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)") + print() + parser.print_help(sys.stderr) + sys.exit(21) + else: + parser.print_help(sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj index 6819ac0eeed95f..c7d63909ee2453 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -263,6 +263,7 @@ runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n"; + showEnvVarsInLog = 0; }; 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = { isa = PBXShellScriptBuildPhase; @@ -282,6 +283,7 @@ runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\necho \"Install app package extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app_packages\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app_packages/ \"$FULL_EXT\"\ndone\necho \"Install app extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m index db00d43da85cbc..ac78456a61e65e 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -50,6 +50,8 @@ - (void)testPython { // Enforce UTF-8 encoding for stderr, stdout, file-system encoding and locale. // See https://docs.python.org/3/library/os.html#python-utf-8-mode. preconfig.utf8_mode = 1; + // Use the system logger for stdout/err + config.use_system_logger = 1; // Don't buffer stdio. We want output to appears in the log immediately config.buffered_stdio = 0; // Don't write bytecode; we can't modify the app bundle From 5eb7fd4d0fc37b91058086181afebec41e66e5ad Mon Sep 17 00:00:00 2001 From: Wulian <1055917385@qq.com> Date: Mon, 9 Dec 2024 20:24:26 +0800 Subject: [PATCH 145/427] gh-127732: Add Windows Server 2025 detection to platform module (GH-127733) --- Lib/platform.py | 3 ++- .../Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst diff --git a/Lib/platform.py b/Lib/platform.py index 239e660cd1621d..1f6baed66d3df9 100644 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -353,7 +353,8 @@ def _wmi_query(table, *keys): ] _WIN32_SERVER_RELEASES = [ - ((10, 1, 0), "post2022Server"), + ((10, 1, 0), "post2025Server"), + ((10, 0, 26100), "2025Server"), ((10, 0, 20348), "2022Server"), ((10, 0, 17763), "2019Server"), ((6, 4, 0), "2016Server"), diff --git a/Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst b/Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst new file mode 100644 index 00000000000000..44821300f6e4e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst @@ -0,0 +1 @@ +The :mod:`platform` module now correctly detects Windows Server 2025. From e85f2f1703e0f79cfd0d0e3010190b71c0eb18da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 9 Dec 2024 19:02:22 +0100 Subject: [PATCH 146/427] gh-127637: add tests for `dis` command-line interface (#127759) --- Lib/dis.py | 4 +- Lib/test/test_dis.py | 122 +++++++++++++++++- ...-12-09-12-35-44.gh-issue-127637.KLx-9I.rst | 1 + 3 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst diff --git a/Lib/dis.py b/Lib/dis.py index 6b3e9ef8399e1c..aa22404c6687e1 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -1115,7 +1115,7 @@ def dis(self): return output.getvalue() -def main(): +def main(args=None): import argparse parser = argparse.ArgumentParser() @@ -1128,7 +1128,7 @@ def main(): parser.add_argument('-S', '--specialized', action='store_true', help='show specialized bytecode') parser.add_argument('infile', nargs='?', default='-') - args = parser.parse_args() + args = parser.parse_args(args=args) if args.infile == '-': name = '' source = sys.stdin.buffer.read() diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 55890e58ed4bae..c719f571152d61 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -5,15 +5,19 @@ import dis import functools import io +import itertools +import opcode import re import sys +import tempfile +import textwrap import types import unittest from test.support import (captured_stdout, requires_debug_ranges, - requires_specialization, cpython_only) + requires_specialization, cpython_only, + os_helper) from test.support.bytecode_helper import BytecodeTestCase -import opcode CACHE = dis.opmap["CACHE"] @@ -2426,5 +2430,119 @@ def _unroll_caches_as_Instructions(instrs, show_caches=False): False, None, None, instr.positions) +class TestDisCLI(unittest.TestCase): + + def setUp(self): + self.filename = tempfile.mktemp() + self.addCleanup(os_helper.unlink, self.filename) + + @staticmethod + def text_normalize(string): + """Dedent *string* and strip it from its surrounding whitespaces. + + This method is used by the other utility functions so that any + string to write or to match against can be freely indented. + """ + return textwrap.dedent(string).strip() + + def set_source(self, content): + with open(self.filename, 'w') as fp: + fp.write(self.text_normalize(content)) + + def invoke_dis(self, *flags): + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.main(args=[*flags, self.filename]) + return self.text_normalize(output.getvalue()) + + def check_output(self, source, expect, *flags): + with self.subTest(source=source, flags=flags): + self.set_source(source) + res = self.invoke_dis(*flags) + expect = self.text_normalize(expect) + self.assertListEqual(res.splitlines(), expect.splitlines()) + + def test_invocation(self): + # test various combinations of parameters + base_flags = [ + ('-C', '--show-caches'), + ('-O', '--show-offsets'), + ('-P', '--show-positions'), + ('-S', '--specialized'), + ] + + self.set_source(''' + def f(): + print(x) + return None + ''') + + for r in range(1, len(base_flags) + 1): + for choices in itertools.combinations(base_flags, r=r): + for args in itertools.product(*choices): + with self.subTest(args=args[1:]): + _ = self.invoke_dis(*args) + + with self.assertRaises(SystemExit): + # suppress argparse error message + with contextlib.redirect_stderr(io.StringIO()): + _ = self.invoke_dis('--unknown') + + def test_show_cache(self): + # test 'python -m dis -C/--show-caches' + source = 'print()' + expect = ''' + 0 RESUME 0 + + 1 LOAD_NAME 0 (print) + PUSH_NULL + CALL 0 + CACHE 0 (counter: 0) + CACHE 0 (func_version: 0) + CACHE 0 + POP_TOP + LOAD_CONST 0 (None) + RETURN_VALUE + ''' + for flag in ['-C', '--show-caches']: + self.check_output(source, expect, flag) + + def test_show_offsets(self): + # test 'python -m dis -O/--show-offsets' + source = 'pass' + expect = ''' + 0 0 RESUME 0 + + 1 2 LOAD_CONST 0 (None) + 4 RETURN_VALUE + ''' + for flag in ['-O', '--show-offsets']: + self.check_output(source, expect, flag) + + def test_show_positions(self): + # test 'python -m dis -P/--show-positions' + source = 'pass' + expect = ''' + 0:0-1:0 RESUME 0 + + 1:0-1:4 LOAD_CONST 0 (None) + 1:0-1:4 RETURN_VALUE + ''' + for flag in ['-P', '--show-positions']: + self.check_output(source, expect, flag) + + def test_specialized_code(self): + # test 'python -m dis -S/--specialized' + source = 'pass' + expect = ''' + 0 RESUME 0 + + 1 LOAD_CONST_IMMORTAL 0 (None) + RETURN_VALUE + ''' + for flag in ['-S', '--specialized']: + self.check_output(source, expect, flag) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst b/Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst new file mode 100644 index 00000000000000..ac5d9827b07199 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst @@ -0,0 +1 @@ +Add tests for the :mod:`dis` command-line interface. Patch by Bénédikt Tran. From 5c89adf385aaaca97c2ee9074f8b1fda0f57ad26 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Mon, 9 Dec 2024 18:31:22 +0000 Subject: [PATCH 147/427] GH-127456: pathlib ABCs: add protocol for path parser (#127494) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the default value of `PurePathBase.parser` from `ParserBase()` to `posixpath`. As a result, user subclasses of `PurePathBase` and `PathBase` use POSIX path syntax by default, which is very often desirable. Move `pathlib._abc.ParserBase` to `pathlib._types.Parser`, and convert it to a runtime-checkable protocol. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/pathlib/_abc.py | 56 +-------------------- Lib/pathlib/_types.py | 22 ++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 61 +++-------------------- 3 files changed, 32 insertions(+), 107 deletions(-) create mode 100644 Lib/pathlib/_types.py diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 309eab2ff855c3..f68685f21d6d79 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -13,6 +13,7 @@ import functools import operator +import posixpath from errno import EINVAL from glob import _GlobberBase, _no_recurse_symlinks from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO @@ -33,59 +34,6 @@ def _is_case_sensitive(parser): return parser.normcase('Aa') == 'Aa' - -class ParserBase: - """Base class for path parsers, which do low-level path manipulation. - - Path parsers provide a subset of the os.path API, specifically those - functions needed to provide PurePathBase functionality. Each PurePathBase - subclass references its path parser via a 'parser' class attribute. - - Every method in this base class raises an UnsupportedOperation exception. - """ - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported" - - @property - def sep(self): - """The character used to separate path components.""" - raise UnsupportedOperation(self._unsupported_msg('sep')) - - def join(self, path, *paths): - """Join path segments.""" - raise UnsupportedOperation(self._unsupported_msg('join()')) - - def split(self, path): - """Split the path into a pair (head, tail), where *head* is everything - before the final path separator, and *tail* is everything after. - Either part may be empty. - """ - raise UnsupportedOperation(self._unsupported_msg('split()')) - - def splitdrive(self, path): - """Split the path into a 2-item tuple (drive, tail), where *drive* is - a device name or mount point, and *tail* is everything after the - drive. Either part may be empty.""" - raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) - - def splitext(self, path): - """Split the path into a pair (root, ext), where *ext* is empty or - begins with a period and contains at most one period, - and *root* is everything before the extension.""" - raise UnsupportedOperation(self._unsupported_msg('splitext()')) - - def normcase(self, path): - """Normalize the case of the path.""" - raise UnsupportedOperation(self._unsupported_msg('normcase()')) - - def isabs(self, path): - """Returns whether the path is absolute, i.e. unaffected by the - current directory or drive.""" - raise UnsupportedOperation(self._unsupported_msg('isabs()')) - - class PathGlobber(_GlobberBase): """ Class providing shell-style globbing for path objects. @@ -115,7 +63,7 @@ class PurePathBase: # the `__init__()` method. '_raw_paths', ) - parser = ParserBase() + parser = posixpath _globber = PathGlobber def __init__(self, *args): diff --git a/Lib/pathlib/_types.py b/Lib/pathlib/_types.py new file mode 100644 index 00000000000000..60df94d0b46049 --- /dev/null +++ b/Lib/pathlib/_types.py @@ -0,0 +1,22 @@ +""" +Protocols for supporting classes in pathlib. +""" +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class Parser(Protocol): + """Protocol for path parsers, which do low-level path manipulation. + + Path parsers provide a subset of the os.path API, specifically those + functions needed to provide PurePathBase functionality. Each PurePathBase + subclass references its path parser via a 'parser' class attribute. + """ + + sep: str + def join(self, path: str, *paths: str) -> str: ... + def split(self, path: str) -> tuple[str, str]: ... + def splitdrive(self, path: str) -> tuple[str, str]: ... + def splitext(self, path: str) -> tuple[str, str]: ... + def normcase(self, path: str) -> str: ... + def isabs(self, path: str) -> bool: ... diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 675abf30a9f13c..dd9425ce393623 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -5,7 +5,8 @@ import stat import unittest -from pathlib._abc import UnsupportedOperation, ParserBase, PurePathBase, PathBase +from pathlib._abc import UnsupportedOperation, PurePathBase, PathBase +from pathlib._types import Parser import posixpath from test.support.os_helper import TESTFN @@ -31,22 +32,6 @@ def test_is_notimplemented(self): self.assertTrue(issubclass(UnsupportedOperation, NotImplementedError)) self.assertTrue(isinstance(UnsupportedOperation(), NotImplementedError)) - -class ParserBaseTest(unittest.TestCase): - cls = ParserBase - - def test_unsupported_operation(self): - m = self.cls() - e = UnsupportedOperation - with self.assertRaises(e): - m.sep - self.assertRaises(e, m.join, 'foo') - self.assertRaises(e, m.split, 'foo') - self.assertRaises(e, m.splitdrive, 'foo') - self.assertRaises(e, m.splitext, 'foo') - self.assertRaises(e, m.normcase, 'foo') - self.assertRaises(e, m.isabs, 'foo') - # # Tests for the pure classes. # @@ -55,37 +40,6 @@ def test_unsupported_operation(self): class PurePathBaseTest(unittest.TestCase): cls = PurePathBase - def test_unsupported_operation_pure(self): - p = self.cls('foo') - e = UnsupportedOperation - with self.assertRaises(e): - p.drive - with self.assertRaises(e): - p.root - with self.assertRaises(e): - p.anchor - with self.assertRaises(e): - p.parts - with self.assertRaises(e): - p.parent - with self.assertRaises(e): - p.parents - with self.assertRaises(e): - p.name - with self.assertRaises(e): - p.stem - with self.assertRaises(e): - p.suffix - with self.assertRaises(e): - p.suffixes - self.assertRaises(e, p.with_name, 'bar') - self.assertRaises(e, p.with_stem, 'bar') - self.assertRaises(e, p.with_suffix, '.txt') - self.assertRaises(e, p.relative_to, '') - self.assertRaises(e, p.is_relative_to, '') - self.assertRaises(e, p.is_absolute) - self.assertRaises(e, p.match, '*') - def test_magic_methods(self): P = self.cls self.assertFalse(hasattr(P, '__fspath__')) @@ -100,12 +54,11 @@ def test_magic_methods(self): self.assertIs(P.__ge__, object.__ge__) def test_parser(self): - self.assertIsInstance(self.cls.parser, ParserBase) + self.assertIs(self.cls.parser, posixpath) class DummyPurePath(PurePathBase): __slots__ = () - parser = posixpath def __eq__(self, other): if not isinstance(other, DummyPurePath): @@ -136,6 +89,9 @@ def setUp(self): self.sep = self.parser.sep self.altsep = self.parser.altsep + def test_parser(self): + self.assertIsInstance(self.cls.parser, Parser) + def test_constructor_common(self): P = self.cls p = P('a') @@ -1359,8 +1315,8 @@ def test_unsupported_operation(self): self.assertRaises(e, p.write_bytes, b'foo') self.assertRaises(e, p.write_text, 'foo') self.assertRaises(e, p.iterdir) - self.assertRaises(e, p.glob, '*') - self.assertRaises(e, p.rglob, '*') + self.assertRaises(e, lambda: list(p.glob('*'))) + self.assertRaises(e, lambda: list(p.rglob('*'))) self.assertRaises(e, lambda: list(p.walk())) self.assertRaises(e, p.expanduser) self.assertRaises(e, p.readlink) @@ -1411,7 +1367,6 @@ class DummyPath(PathBase): memory. """ __slots__ = () - parser = posixpath _files = {} _directories = {} From 3b18af964da9814474a5db9e502962c7e0593e8d Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 10 Dec 2024 03:32:58 +0100 Subject: [PATCH 148/427] gh-127629: Add ctypes to the Emscripten build (#127683) Adds tooling to build libffi and add ctypes to the stdlib for Emscripten. --- ...-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst | 1 + Tools/wasm/README.md | 1 + Tools/wasm/emscripten/__main__.py | 64 +++++++++++++++---- Tools/wasm/emscripten/make_libffi.sh | 21 ++++++ 4 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst create mode 100755 Tools/wasm/emscripten/make_libffi.sh diff --git a/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst b/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst new file mode 100644 index 00000000000000..52ee84f5d6f6c3 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst @@ -0,0 +1 @@ +Emscripten builds now include ctypes support. diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 4802d9683de52e..2e0fa2fb533d67 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -61,6 +61,7 @@ or you can break it out into four separate steps: ```shell python Tools/wasm/emscripten configure-build-python python Tools/wasm/emscripten make-build-python +python Tools/wasm/emscripten make-libffi python Tools/wasm/emscripten configure-host python Tools/wasm/emscripten make-host ``` diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 6843b6fdeceb8c..4a53e0bd1bee1b 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -9,6 +9,7 @@ import sys import sysconfig import tempfile +from urllib.request import urlopen from pathlib import Path from textwrap import dedent @@ -22,9 +23,13 @@ CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent CROSS_BUILD_DIR = CHECKOUT / "cross-build" -BUILD_DIR = CROSS_BUILD_DIR / "build" +NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build" HOST_TRIPLE = "wasm32-emscripten" -HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE + +DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" +HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" +HOST_DIR = HOST_BUILD_DIR / "python" +PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8") @@ -118,16 +123,16 @@ def build_platform(): def build_python_path(): """The path to the build Python binary.""" - binary = BUILD_DIR / "python" + binary = NATIVE_BUILD_DIR / "python" if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " f"{BUILD_DIR}") + raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}") return binary -@subdir(BUILD_DIR, clean_ok=True) +@subdir(NATIVE_BUILD_DIR, clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" if LOCAL_SETUP.exists(): @@ -143,7 +148,7 @@ def configure_build_python(context, working_dir): call(configure, quiet=context.quiet) -@subdir(BUILD_DIR) +@subdir(NATIVE_BUILD_DIR) def make_build_python(context, working_dir): """Make/build the build Python.""" call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) @@ -159,6 +164,23 @@ def make_build_python(context, working_dir): print(f"🎉 {binary} {version}") +@subdir(HOST_BUILD_DIR, clean_ok=True) +def make_emscripten_libffi(context, working_dir): + shutil.rmtree(working_dir / "libffi-3.4.6", ignore_errors=True) + with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tmp_file: + with urlopen( + "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz" + ) as response: + shutil.copyfileobj(response, tmp_file) + shutil.unpack_archive(tmp_file.name, working_dir) + call( + [EMSCRIPTEN_DIR / "make_libffi.sh"], + env=updated_env({"PREFIX": PREFIX_DIR}), + cwd=working_dir / "libffi-3.4.6", + quiet=context.quiet, + ) + + @subdir(HOST_DIR, clean_ok=True) def configure_emscripten_python(context, working_dir): """Configure the emscripten/host build.""" @@ -168,7 +190,7 @@ def configure_emscripten_python(context, working_dir): emscripten_build_dir = working_dir.relative_to(CHECKOUT) - python_build_dir = BUILD_DIR / "build" + python_build_dir = NATIVE_BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) assert ( len(lib_dirs) == 1 @@ -183,12 +205,18 @@ def configure_emscripten_python(context, working_dir): sysconfig_data += "-pydebug" host_runner = context.host_runner - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve() + env_additions = { + "CONFIG_SITE": config_site, + "HOSTRUNNER": host_runner, + "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir), + } build_python = os.fsdecode(build_python_path()) configure = [ "emconfigure", os.path.relpath(CHECKOUT / "configure", working_dir), "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", + "PKG_CONFIG=pkg-config", f"--host={HOST_TRIPLE}", f"--build={build_platform()}", f"--with-build-python={build_python}", @@ -197,7 +225,7 @@ def configure_emscripten_python(context, working_dir): "--disable-ipv6", "--enable-big-digits=30", "--enable-wasm-dynamic-linking", - f"--prefix={HOST_DIR}", + f"--prefix={PREFIX_DIR}", ] if pydebug: configure.append("--with-pydebug") @@ -264,6 +292,7 @@ def build_all(context): steps = [ configure_build_python, make_build_python, + make_emscripten_libffi, configure_emscripten_python, make_emscripten_python, ] @@ -292,6 +321,9 @@ def main(): configure_build = subcommands.add_parser( "configure-build-python", help="Run `configure` for the " "build Python" ) + make_libffi_cmd = subcommands.add_parser( + "make-libffi", help="Clone libffi repo, configure and build it for emscripten" + ) make_build = subcommands.add_parser( "make-build-python", help="Run `make` for the build Python" ) @@ -299,11 +331,20 @@ def main(): "configure-host", help="Run `configure` for the host/emscripten (pydebug builds are inferred from the build Python)", ) - make_host = subcommands.add_parser("make-host", help="Run `make` for the host/emscripten") + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/emscripten" + ) clean = subcommands.add_parser( "clean", help="Delete files and directories created by this script" ) - for subcommand in build, configure_build, make_build, configure_host, make_host: + for subcommand in ( + build, + configure_build, + make_libffi_cmd, + make_build, + configure_host, + make_host, + ): subcommand.add_argument( "--quiet", action="store_true", @@ -336,6 +377,7 @@ def main(): context = parser.parse_args() dispatch = { + "make-libffi": make_emscripten_libffi, "configure-build-python": configure_build_python, "make-build-python": make_build_python, "configure-host": configure_emscripten_python, diff --git a/Tools/wasm/emscripten/make_libffi.sh b/Tools/wasm/emscripten/make_libffi.sh new file mode 100755 index 00000000000000..3c75c4d5127102 --- /dev/null +++ b/Tools/wasm/emscripten/make_libffi.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set +e + +export CFLAGS="-O2 -fPIC -DWASM_BIGINT" +export CXXFLAGS="$CFLAGS" + +# Build paths +export CPATH="$PREFIX/include" +export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig" +export EM_PKG_CONFIG_PATH="$PKG_CONFIG_PATH" + +# Specific variables for cross-compilation +export CHOST="wasm32-unknown-linux" # wasm32-unknown-emscripten + +emconfigure ./configure --host=$CHOST --prefix="$PREFIX" --enable-static --disable-shared --disable-dependency-tracking \ + --disable-builddir --disable-multi-os-directory --disable-raw-api --disable-docs + +make install +# Some forgotten headers? +cp fficonfig.h $PREFIX/include/ +cp include/ffi_common.h $PREFIX/include/ From 58c753827ac7aa3d7f1495ac206c28bf2f6c67e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 10 Dec 2024 03:48:38 +0100 Subject: [PATCH 149/427] gh-125420: implement `Sequence.index` API on `memoryview` objects (#125446) --- Doc/library/stdtypes.rst | 9 ++ Lib/test/test_memoryview.py | 26 ++++++ ...-10-14-13-28-16.gh-issue-125420.hNKixM.rst | 2 + Objects/clinic/memoryobject.c.h | 48 +++++++++- Objects/memoryobject.c | 87 +++++++++++++++++++ 5 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 4f4fc9fba63120..f4e635dd61e3dc 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4149,6 +4149,15 @@ copying. .. versionchanged:: 3.5 The source format is no longer restricted when casting to a byte view. + .. method:: index(value, start=0, stop=sys.maxsize, /) + + Return the index of the first occurrence of *value* (at or after + index *start* and before index *stop*). + + Raises a :exc:`ValueError` if *value* cannot be found. + + .. versionadded:: next + There are also several readonly attributes available: .. attribute:: obj diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 2d4bf5f1408df8..8f07828c19790d 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -15,6 +15,7 @@ import pickle import struct +from itertools import product from test.support import import_helper @@ -58,6 +59,31 @@ def test_getitem(self): for tp in self._types: self.check_getitem_with_type(tp) + def test_index(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) # may be a sub-view + l = m.tolist() + k = 2 * len(self._source) + + for chi in self._source: + if chi in l: + self.assertEqual(m.index(chi), l.index(chi)) + else: + self.assertRaises(ValueError, m.index, chi) + + for start, stop in product(range(-k, k), range(-k, k)): + index = -1 + try: + index = l.index(chi, start, stop) + except ValueError: + pass + + if index == -1: + self.assertRaises(ValueError, m.index, chi, start, stop) + else: + self.assertEqual(m.index(chi, start, stop), index) + def test_iter(self): for tp in self._types: b = tp(self._source) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst new file mode 100644 index 00000000000000..6ed823175f6754 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst @@ -0,0 +1,2 @@ +Add :meth:`memoryview.index` to :class:`memoryview` objects. Patch by +Bénédikt Tran. diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 185d7819b6b84a..9b8a23317f0da0 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -418,4 +418,50 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=0a93f08110630633 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(memoryview_index__doc__, +"index($self, value, start=0, stop=sys.maxsize, /)\n" +"--\n" +"\n" +"Return the index of the first occurrence of a value.\n" +"\n" +"Raises ValueError if the value is not present."); + +#define MEMORYVIEW_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(memoryview_index), METH_FASTCALL, memoryview_index__doc__}, + +static PyObject * +memoryview_index_impl(PyMemoryViewObject *self, PyObject *value, + Py_ssize_t start, Py_ssize_t stop); + +static PyObject * +memoryview_index(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *value; + Py_ssize_t start = 0; + Py_ssize_t stop = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + value = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[2], &stop)) { + goto exit; + } +skip_optional: + return_value = memoryview_index_impl(self, value, start, stop); + +exit: + return return_value; +} +/*[clinic end generated code: output=2742d371dba7314f input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 25634f997ac66b..345aa79c328125 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2748,6 +2748,92 @@ static PySequenceMethods memory_as_sequence = { }; +/**************************************************************************/ +/* Lookup */ +/**************************************************************************/ + +/*[clinic input] +memoryview.index + + value: object + start: slice_index(accept={int}) = 0 + stop: slice_index(accept={int}, c_default="PY_SSIZE_T_MAX") = sys.maxsize + / + +Return the index of the first occurrence of a value. + +Raises ValueError if the value is not present. +[clinic start generated code]*/ + +static PyObject * +memoryview_index_impl(PyMemoryViewObject *self, PyObject *value, + Py_ssize_t start, Py_ssize_t stop) +/*[clinic end generated code: output=e0185e3819e549df input=0697a0165bf90b5a]*/ +{ + const Py_buffer *view = &self->view; + CHECK_RELEASED(self); + + if (view->ndim == 0) { + PyErr_SetString(PyExc_TypeError, "invalid lookup on 0-dim memory"); + return NULL; + } + + if (view->ndim == 1) { + Py_ssize_t n = view->shape[0]; + + if (start < 0) { + start = Py_MAX(start + n, 0); + } + + if (stop < 0) { + stop = Py_MAX(stop + n, 0); + } + + stop = Py_MIN(stop, n); + assert(stop >= 0); + assert(stop <= n); + + start = Py_MIN(start, stop); + assert(0 <= start); + assert(start <= stop); + + PyObject *obj = _PyObject_CAST(self); + for (Py_ssize_t index = start; index < stop; index++) { + // Note: while memoryviews can be mutated during iterations + // when calling the == operator, their shape cannot. As such, + // it is safe to assume that the index remains valid for the + // entire loop. + assert(index < n); + + PyObject *item = memory_item(obj, index); + if (item == NULL) { + return NULL; + } + if (item == value) { + Py_DECREF(item); + return PyLong_FromSsize_t(index); + } + int contained = PyObject_RichCompareBool(item, value, Py_EQ); + Py_DECREF(item); + if (contained > 0) { // more likely than 'contained < 0' + return PyLong_FromSsize_t(index); + } + else if (contained < 0) { + return NULL; + } + } + + PyErr_SetString(PyExc_ValueError, "memoryview.index(x): x not found"); + return NULL; + } + + PyErr_SetString(PyExc_NotImplementedError, + "multi-dimensional lookup is not implemented"); + return NULL; + +} + + /**************************************************************************/ /* Comparisons */ /**************************************************************************/ @@ -3284,6 +3370,7 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_CAST_METHODDEF MEMORYVIEW_TOREADONLY_METHODDEF MEMORYVIEW__FROM_FLAGS_METHODDEF + MEMORYVIEW_INDEX_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__", memory_exit, METH_VARARGS, memory_exit_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, From 3983527c3a6b389e373a233e514919555853ccb3 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 9 Dec 2024 20:55:20 -0800 Subject: [PATCH 150/427] gh-127651: Use __file__ in diagnostics if origin is missing (#127660) See the left hand side in https://github.com/python/cpython/pull/123929/files#diff-c22186367cbe20233e843261998dc027ae5f1f8c0d2e778abfa454ae74cc59deL2840-L2849 --------- Co-authored-by: Serhiy Storchaka --- Lib/test/test_import/__init__.py | 48 ++++++++++++++++++- ...-12-06-01-09-40.gh-issue-127651.80cm6j.rst | 1 + Python/ceval.c | 22 +++++++-- 3 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index c52b7f3e09bea1..83efbc1e25e77a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -807,6 +807,50 @@ def test_issue105979(self): self.assertIn("Frozen object named 'x' is invalid", str(cm.exception)) + def test_frozen_module_from_import_error(self): + with self.assertRaises(ImportError) as cm: + from os import this_will_never_exist + self.assertIn( + f"cannot import name 'this_will_never_exist' from 'os' ({os.__file__})", + str(cm.exception), + ) + with self.assertRaises(ImportError) as cm: + from sys import this_will_never_exist + self.assertIn( + "cannot import name 'this_will_never_exist' from 'sys' (unknown location)", + str(cm.exception), + ) + + scripts = [ + """ +import os +os.__spec__.has_location = False +os.__file__ = [] +from os import this_will_never_exist +""", + """ +import os +os.__spec__.has_location = False +del os.__file__ +from os import this_will_never_exist +""", + """ +import os +os.__spec__.origin = [] +os.__file__ = [] +from os import this_will_never_exist +""" + ] + for script in scripts: + with self.subTest(script=script): + expected_error = ( + b"cannot import name 'this_will_never_exist' " + b"from 'os' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + def test_script_shadowing_stdlib(self): script_errors = [ ( @@ -1068,7 +1112,7 @@ class substr(str): except AttributeError as e: print(str(e)) -fractions.__spec__.origin = 0 +fractions.__spec__.origin = [] try: fractions.Fraction except AttributeError as e: @@ -1092,7 +1136,7 @@ class substr(str): except ImportError as e: print(str(e)) -fractions.__spec__.origin = 0 +fractions.__spec__.origin = [] try: from fractions import Fraction except ImportError as e: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst new file mode 100644 index 00000000000000..92b18b082eb30e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst @@ -0,0 +1 @@ +When raising :exc:`ImportError` for missing symbols in ``from`` imports, use ``__file__`` in the error message if ``__spec__.origin`` is not a location diff --git a/Python/ceval.c b/Python/ceval.c index 6795a160506231..5eda033eced628 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2859,6 +2859,20 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) } } + if (origin == NULL) { + // Fall back to __file__ for diagnostics if we don't have + // an origin that is a location + origin = PyModule_GetFilenameObject(v); + if (origin == NULL) { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + goto done; + } + // PyModule_GetFilenameObject raised "module filename missing" + _PyErr_Clear(tstate); + } + assert(origin == NULL || PyUnicode_Check(origin)); + } + if (is_possibly_shadowing_stdlib) { assert(origin); errmsg = PyUnicode_FromFormat( @@ -2919,9 +2933,11 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) } done_with_errmsg: - /* NULL checks for errmsg, mod_name, origin done by PyErr_SetImportError. */ - _PyErr_SetImportErrorWithNameFrom(errmsg, mod_name, origin, name); - Py_DECREF(errmsg); + if (errmsg != NULL) { + /* NULL checks for mod_name and origin done by _PyErr_SetImportErrorWithNameFrom */ + _PyErr_SetImportErrorWithNameFrom(errmsg, mod_name, origin, name); + Py_DECREF(errmsg); + } done: Py_XDECREF(origin); From 2233c303e476496fc4c85a29a1429a7e4b1f707b Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 10 Dec 2024 07:40:54 +0000 Subject: [PATCH 151/427] gh-126775: make linecache.checkcache threadsafe and GC re-entrency safe (#126776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Bartosz Sławecki --- Lib/linecache.py | 13 ++++++++----- .../2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst diff --git a/Lib/linecache.py b/Lib/linecache.py index 4b38a0464d8747..8ba2df73d5a8fb 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -49,14 +49,17 @@ def checkcache(filename=None): (This is not checked upon each call!)""" if filename is None: - filenames = list(cache.keys()) - elif filename in cache: - filenames = [filename] + # get keys atomically + filenames = cache.copy().keys() else: - return + filenames = [filename] for filename in filenames: - entry = cache[filename] + try: + entry = cache[filename] + except KeyError: + continue + if len(entry) == 1: # lazy cache entry, leave it lazy. continue diff --git a/Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst b/Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst new file mode 100644 index 00000000000000..429fc2aa17bd42 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst @@ -0,0 +1 @@ +Make :func:`linecache.checkcache` thread safe and GC re-entrancy safe. From 212448b1623b45f19c5529595e081da72a6521a0 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 10 Dec 2024 09:44:15 +0200 Subject: [PATCH 152/427] gh-127718: Add colour to `test.regrtest` output (#127719) --- Doc/library/test.rst | 4 + Lib/test/libregrtest/main.py | 9 +- Lib/test/libregrtest/result.py | 50 +++++++---- Lib/test/libregrtest/results.py | 89 +++++++++++++------ Lib/test/libregrtest/single.py | 22 +++-- Lib/test/test_regrtest.py | 46 ++++++++++ ...-12-07-15-28-31.gh-issue-127718.9dpLfi.rst | 1 + 7 files changed, 168 insertions(+), 53 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 04d28aee0f8672..b5b6e442e218fd 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -192,6 +192,10 @@ top-level directory where Python was built. On Windows, executing :program:`rt.bat` from your :file:`PCbuild` directory will run all regression tests. +.. versionadded:: 3.14 + Output is colorized by default and can be + :ref:`controlled using environment variables `. + :mod:`test.support` --- Utilities for the Python test suite =========================================================== diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 49209b0cec756e..dcbcc6790c68d8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -6,6 +6,7 @@ import sysconfig import time import trace +from _colorize import get_colors # type: ignore[import-not-found] from typing import NoReturn from test.support import os_helper, MS_WINDOWS, flush_std_streams @@ -270,6 +271,9 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: return runtests def rerun_failed_tests(self, runtests: RunTests) -> None: + ansi = get_colors() + red, reset = ansi.BOLD_RED, ansi.RESET + if self.python_cmd: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( @@ -284,7 +288,10 @@ def rerun_failed_tests(self, runtests: RunTests) -> None: rerun_runtests = self._rerun_failed_tests(runtests) if self.results.bad: - print(count(len(self.results.bad), 'test'), "failed again:") + print( + f"{red}{count(len(self.results.bad), 'test')} " + f"failed again:{reset}" + ) printlist(self.results.bad) self.display_result(rerun_runtests) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 7553efe5e8abeb..daf7624366ee20 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,5 +1,6 @@ import dataclasses import json +from _colorize import get_colors # type: ignore[import-not-found] from typing import Any from .utils import ( @@ -105,54 +106,71 @@ def is_failed(self, fail_env_changed: bool) -> bool: return State.is_failed(self.state) def _format_failed(self): + ansi = get_colors() + red, reset = ansi.BOLD_RED, ansi.RESET if self.errors and self.failures: le = len(self.errors) lf = len(self.failures) error_s = "error" + ("s" if le > 1 else "") failure_s = "failure" + ("s" if lf > 1 else "") - return f"{self.test_name} failed ({le} {error_s}, {lf} {failure_s})" + return ( + f"{red}{self.test_name} failed " + f"({le} {error_s}, {lf} {failure_s}){reset}" + ) if self.errors: le = len(self.errors) error_s = "error" + ("s" if le > 1 else "") - return f"{self.test_name} failed ({le} {error_s})" + return f"{red}{self.test_name} failed ({le} {error_s}){reset}" if self.failures: lf = len(self.failures) failure_s = "failure" + ("s" if lf > 1 else "") - return f"{self.test_name} failed ({lf} {failure_s})" + return f"{red}{self.test_name} failed ({lf} {failure_s}){reset}" - return f"{self.test_name} failed" + return f"{red}{self.test_name} failed{reset}" def __str__(self) -> str: + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + match self.state: case State.PASSED: - return f"{self.test_name} passed" + return f"{green}{self.test_name} passed{reset}" case State.FAILED: - return self._format_failed() + return f"{red}{self._format_failed()}{reset}" case State.SKIPPED: - return f"{self.test_name} skipped" + return f"{yellow}{self.test_name} skipped{reset}" case State.UNCAUGHT_EXC: - return f"{self.test_name} failed (uncaught exception)" + return ( + f"{red}{self.test_name} failed (uncaught exception){reset}" + ) case State.REFLEAK: - return f"{self.test_name} failed (reference leak)" + return f"{red}{self.test_name} failed (reference leak){reset}" case State.ENV_CHANGED: - return f"{self.test_name} failed (env changed)" + return f"{red}{self.test_name} failed (env changed){reset}" case State.RESOURCE_DENIED: - return f"{self.test_name} skipped (resource denied)" + return f"{yellow}{self.test_name} skipped (resource denied){reset}" case State.INTERRUPTED: - return f"{self.test_name} interrupted" + return f"{yellow}{self.test_name} interrupted{reset}" case State.WORKER_FAILED: - return f"{self.test_name} worker non-zero exit code" + return ( + f"{red}{self.test_name} worker non-zero exit code{reset}" + ) case State.WORKER_BUG: - return f"{self.test_name} worker bug" + return f"{red}{self.test_name} worker bug{reset}" case State.DID_NOT_RUN: - return f"{self.test_name} ran no tests" + return f"{yellow}{self.test_name} ran no tests{reset}" case State.TIMEOUT: assert self.duration is not None, "self.duration is None" return f"{self.test_name} timed out ({format_duration(self.duration)})" case _: - raise ValueError("unknown result state: {state!r}") + raise ValueError( + f"{red}unknown result state: {{state!r}}{reset}" + ) def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 9eda926966dc7e..a35934fc2c9ca8 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,5 +1,6 @@ import sys import trace +from _colorize import get_colors # type: ignore[import-not-found] from typing import TYPE_CHECKING from .runtests import RunTests @@ -59,19 +60,24 @@ def no_tests_run(self) -> bool: def get_state(self, fail_env_changed: bool) -> str: state = [] + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW if self.bad: - state.append("FAILURE") + state.append(f"{red}FAILURE{reset}") elif fail_env_changed and self.env_changed: - state.append("ENV CHANGED") + state.append(f"{yellow}ENV CHANGED{reset}") elif self.no_tests_run(): - state.append("NO TESTS RAN") + state.append(f"{yellow}NO TESTS RAN{reset}") if self.interrupted: - state.append("INTERRUPTED") + state.append(f"{yellow}INTERRUPTED{reset}") if self.worker_bug: - state.append("WORKER BUG") + state.append(f"{red}WORKER BUG{reset}") if not state: - state.append("SUCCESS") + state.append(f"{green}SUCCESS{reset}") return ', '.join(state) @@ -197,27 +203,51 @@ def write_junit(self, filename: StrPath) -> None: f.write(s) def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None: + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + if print_slowest: self.test_times.sort(reverse=True) print() - print("10 slowest tests:") + print(f"{yellow}10 slowest tests:{reset}") for test_time, test in self.test_times[:10]: - print("- %s: %s" % (test, format_duration(test_time))) + print(f"- {test}: {format_duration(test_time)}") all_tests = [] omitted = set(tests) - self.get_executed() # less important - all_tests.append((sorted(omitted), "test", "{} omitted:")) + all_tests.append( + (sorted(omitted), "test", f"{yellow}{{}} omitted:{reset}") + ) if not quiet: - all_tests.append((self.skipped, "test", "{} skipped:")) - all_tests.append((self.resource_denied, "test", "{} skipped (resource denied):")) - all_tests.append((self.run_no_tests, "test", "{} run no tests:")) + all_tests.append( + (self.skipped, "test", f"{yellow}{{}} skipped:{reset}") + ) + all_tests.append( + ( + self.resource_denied, + "test", + f"{yellow}{{}} skipped (resource denied):{reset}", + ) + ) + all_tests.append( + (self.run_no_tests, "test", f"{yellow}{{}} run no tests:{reset}") + ) # more important - all_tests.append((self.env_changed, "test", "{} altered the execution environment (env changed):")) - all_tests.append((self.rerun, "re-run test", "{}:")) - all_tests.append((self.bad, "test", "{} failed:")) + all_tests.append( + ( + self.env_changed, + "test", + f"{yellow}{{}} altered the execution environment (env changed):{reset}", + ) + ) + all_tests.append((self.rerun, "re-run test", f"{yellow}{{}}:{reset}")) + all_tests.append((self.bad, "test", f"{red}{{}} failed:{reset}")) for tests_list, count_text, title_format in all_tests: if tests_list: @@ -229,26 +259,29 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> if self.good and not quiet: print() text = count(len(self.good), "test") - text = f"{text} OK." - if (self.is_all_good() and len(self.good) > 1): + text = f"{green}{text} OK.{reset}" + if self.is_all_good() and len(self.good) > 1: text = f"All {text}" print(text) if self.interrupted: print() - print("Test suite interrupted by signal SIGINT.") + print(f"{yellow}Test suite interrupted by signal SIGINT.{reset}") def display_summary(self, first_runtests: RunTests, filtered: bool) -> None: # Total tests + ansi = get_colors() + red, reset, yellow = ansi.RED, ansi.RESET, ansi.YELLOW + stats = self.stats text = f'run={stats.tests_run:,}' if filtered: text = f"{text} (filtered)" report = [text] if stats.failures: - report.append(f'failures={stats.failures:,}') + report.append(f'{red}failures={stats.failures:,}{reset}') if stats.skipped: - report.append(f'skipped={stats.skipped:,}') + report.append(f'{yellow}skipped={stats.skipped:,}{reset}') print(f"Total tests: {' '.join(report)}") # Total test files @@ -263,14 +296,14 @@ def display_summary(self, first_runtests: RunTests, filtered: bool) -> None: if filtered: text = f"{text} (filtered)" report = [text] - for name, tests in ( - ('failed', self.bad), - ('env_changed', self.env_changed), - ('skipped', self.skipped), - ('resource_denied', self.resource_denied), - ('rerun', self.rerun), - ('run_no_tests', self.run_no_tests), + for name, tests, color in ( + ('failed', self.bad, red), + ('env_changed', self.env_changed, yellow), + ('skipped', self.skipped, yellow), + ('resource_denied', self.resource_denied, yellow), + ('rerun', self.rerun, yellow), + ('run_no_tests', self.run_no_tests, yellow), ): if tests: - report.append(f'{name}={len(tests)}') + report.append(f'{color}{name}={len(tests)}{reset}') print(f"Total test files: {' '.join(report)}") diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 17323e7f9cf730..0e174f82abed28 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -7,6 +7,7 @@ import traceback import unittest +from _colorize import get_colors # type: ignore[import-not-found] from test import support from test.support import threading_helper @@ -161,6 +162,8 @@ def test_func(): def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, display_failure: bool = True) -> None: # Handle exceptions, detect environment changes. + ansi = get_colors() + red, reset, yellow = ansi.RED, ansi.RESET, ansi.YELLOW # Reset the environment_altered flag to detect if a test altered # the environment @@ -181,18 +184,18 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, _load_run_test(result, runtests) except support.ResourceDenied as exc: if not quiet and not pgo: - print(f"{test_name} skipped -- {exc}", flush=True) + print(f"{yellow}{test_name} skipped -- {exc}{reset}", flush=True) result.state = State.RESOURCE_DENIED return except unittest.SkipTest as exc: if not quiet and not pgo: - print(f"{test_name} skipped -- {exc}", flush=True) + print(f"{yellow}{test_name} skipped -- {exc}{reset}", flush=True) result.state = State.SKIPPED return except support.TestFailedWithDetails as exc: - msg = f"test {test_name} failed" + msg = f"{red}test {test_name} failed{reset}" if display_failure: - msg = f"{msg} -- {exc}" + msg = f"{red}{msg} -- {exc}{reset}" print(msg, file=sys.stderr, flush=True) result.state = State.FAILED result.errors = exc.errors @@ -200,9 +203,9 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, result.stats = exc.stats return except support.TestFailed as exc: - msg = f"test {test_name} failed" + msg = f"{red}test {test_name} failed{reset}" if display_failure: - msg = f"{msg} -- {exc}" + msg = f"{red}{msg} -- {exc}{reset}" print(msg, file=sys.stderr, flush=True) result.state = State.FAILED result.stats = exc.stats @@ -217,7 +220,7 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, except: if not pgo: msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", + print(f"{red}test {test_name} crashed -- {msg}{reset}", file=sys.stderr, flush=True) result.state = State.UNCAUGHT_EXC return @@ -300,6 +303,9 @@ def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: If runtests.use_junit, xml_data is a list containing each generated testsuite element. """ + ansi = get_colors() + red, reset, yellow = ansi.BOLD_RED, ansi.RESET, ansi.YELLOW + start_time = time.perf_counter() result = TestResult(test_name) pgo = runtests.pgo @@ -308,7 +314,7 @@ def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: except: if not pgo: msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", + print(f"{red}test {test_name} crashed -- {msg}{reset}", file=sys.stderr, flush=True) result.state = State.UNCAUGHT_EXC diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0ab7a23aca1df8..ab46ccbf004a3a 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -4,6 +4,7 @@ Note: test_regrtest cannot be run twice in parallel. """ +import _colorize import contextlib import dataclasses import glob @@ -21,6 +22,7 @@ import tempfile import textwrap import unittest +import unittest.mock from xml.etree import ElementTree from test import support @@ -2487,5 +2489,49 @@ def test_sanitize_xml(self): 'valid t\xe9xt \u20ac') +from test.libregrtest.results import TestResults + + +class TestColorized(unittest.TestCase): + def test_test_result_get_state(self): + # Arrange + green = _colorize.ANSIColors.GREEN + red = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET + yellow = _colorize.ANSIColors.YELLOW + + good_results = TestResults() + good_results.good = ["good1", "good2"] + bad_results = TestResults() + bad_results.bad = ["bad1", "bad2"] + no_results = TestResults() + no_results.bad = [] + interrupted_results = TestResults() + interrupted_results.interrupted = True + interrupted_worker_bug = TestResults() + interrupted_worker_bug.interrupted = True + interrupted_worker_bug.worker_bug = True + + for results, expected in ( + (good_results, f"{green}SUCCESS{reset}"), + (bad_results, f"{red}FAILURE{reset}"), + (no_results, f"{yellow}NO TESTS RAN{reset}"), + (interrupted_results, f"{yellow}INTERRUPTED{reset}"), + ( + interrupted_worker_bug, + f"{yellow}INTERRUPTED{reset}, {red}WORKER BUG{reset}", + ), + ): + with self.subTest(results=results, expected=expected): + # Act + with unittest.mock.patch( + "_colorize.can_colorize", return_value=True + ): + result = results.get_state(fail_env_changed=False) + + # Assert + self.assertEqual(result, expected) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst b/Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst new file mode 100644 index 00000000000000..6c1b7bef610e8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst @@ -0,0 +1 @@ +Add colour to :mod:`test.regrtest` output. Patch by Hugo van Kemenade. From 050d59bd1765de417bf4ec8b5c3cbdac65695f5e Mon Sep 17 00:00:00 2001 From: Skip Montanaro Date: Tue, 10 Dec 2024 02:17:25 -0600 Subject: [PATCH 153/427] add help blurb for "extralargefile" (#127710) --- Lib/test/libregrtest/cmdline.py | 42 +++++++++++++++++---------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 0c94fcc1907071..bf9a71efbdbff9 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -87,38 +87,40 @@ The argument is a comma-separated list of words indicating the resources to test. Currently only the following are defined: - all - Enable all special resources. + all - Enable all special resources. - none - Disable all special resources (this is the default). + none - Disable all special resources (this is the default). - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) - curses - Tests that use curses and will modify the terminal's - state and output modes. + curses - Tests that use curses and will modify the terminal's + state and output modes. - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2 GiB of disk space temporarily. + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2 GiB of disk space temporarily. - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. + extralargefile - Like 'largefile', but even larger (and slower). - decimal - Test the decimal module against a large suite that - verifies compliance with standards. + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. - cpu - Used for certain CPU-heavy tests. + decimal - Test the decimal module against a large suite that + verifies compliance with standards. - walltime - Long running but not CPU-bound tests. + cpu - Used for certain CPU-heavy tests. - subprocess Run all tests for the subprocess module. + walltime - Long running but not CPU-bound tests. - urlfetch - It is okay to download files required on testing. + subprocess Run all tests for the subprocess module. - gui - Run tests that require a running GUI. + urlfetch - It is okay to download files required on testing. - tzdata - Run tests that require timezone data. + gui - Run tests that require a running GUI. + + tzdata - Run tests that require timezone data. To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the gui tests, give the From 4331832db02ff4a7598dcdd99cae31087173dce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:12:33 +0100 Subject: [PATCH 154/427] gh-125420: implement `Sequence.count` API on `memoryview` objects (#125443) --- Doc/library/stdtypes.rst | 8 ++- Lib/test/test_memoryview.py | 28 +++++++++++ ...-10-14-12-34-51.gh-issue-125420.jABXoZ.rst | 2 + Objects/clinic/memoryobject.c.h | 11 +++- Objects/memoryobject.c | 50 +++++++++++++++++++ 5 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index f4e635dd61e3dc..d5f7714ced6873 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4149,7 +4149,13 @@ copying. .. versionchanged:: 3.5 The source format is no longer restricted when casting to a byte view. - .. method:: index(value, start=0, stop=sys.maxsize, /) + .. method:: count(value, /) + + Count the number of occurrences of *value*. + + .. versionadded:: next + + .. method:: index(value, start=0, stop=sys.maxsize, /) Return the index of the first occurrence of *value* (at or after index *start* and before index *stop*). diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 8f07828c19790d..96320ebef6467a 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -90,6 +90,22 @@ def test_iter(self): m = self._view(b) self.assertEqual(list(m), [m[i] for i in range(len(m))]) + def test_count(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) + l = m.tolist() + for ch in list(m): + self.assertEqual(m.count(ch), l.count(ch)) + + b = tp((b'a' * 5) + (b'c' * 3)) + m = self._view(b) # may be sliced + l = m.tolist() + with self.subTest('count', buffer=b): + self.assertEqual(m.count(ord('a')), l.count(ord('a'))) + self.assertEqual(m.count(ord('b')), l.count(ord('b'))) + self.assertEqual(m.count(ord('c')), l.count(ord('c'))) + def test_setitem_readonly(self): if not self.ro_type: self.skipTest("no read-only type to test") @@ -464,6 +480,18 @@ def _view(self, obj): def _check_contents(self, tp, obj, contents): self.assertEqual(obj, tp(contents)) + def test_count(self): + super().test_count() + for tp in self._types: + b = tp((b'a' * 5) + (b'c' * 3)) + m = self._view(b) # should not be sliced + self.assertEqual(len(b), len(m)) + with self.subTest('count', buffer=b): + self.assertEqual(m.count(ord('a')), 5) + self.assertEqual(m.count(ord('b')), 0) + self.assertEqual(m.count(ord('c')), 3) + + class BaseMemorySliceTests: source_bytes = b"XabcdefY" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst new file mode 100644 index 00000000000000..ef120802b5dc59 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst @@ -0,0 +1,2 @@ +Add :meth:`memoryview.count` to :class:`memoryview` objects. Patch by +Bénédikt Tran. diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 9b8a23317f0da0..a6cf1f431a15b0 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -419,6 +419,15 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(memoryview_count__doc__, +"count($self, value, /)\n" +"--\n" +"\n" +"Count the number of occurrences of a value."); + +#define MEMORYVIEW_COUNT_METHODDEF \ + {"count", (PyCFunction)memoryview_count, METH_O, memoryview_count__doc__}, + PyDoc_STRVAR(memoryview_index__doc__, "index($self, value, start=0, stop=sys.maxsize, /)\n" "--\n" @@ -464,4 +473,4 @@ memoryview_index(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=2742d371dba7314f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=132893ef5f67ad73 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 345aa79c328125..afe2dcb127adb4 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2748,6 +2748,55 @@ static PySequenceMethods memory_as_sequence = { }; +/****************************************************************************/ +/* Counting */ +/****************************************************************************/ + +/*[clinic input] +memoryview.count + + value: object + / + +Count the number of occurrences of a value. +[clinic start generated code]*/ + +static PyObject * +memoryview_count(PyMemoryViewObject *self, PyObject *value) +/*[clinic end generated code: output=e2c255a8d54eaa12 input=e3036ce1ed7d1823]*/ +{ + PyObject *iter = PyObject_GetIter(_PyObject_CAST(self)); + if (iter == NULL) { + return NULL; + } + + Py_ssize_t count = 0; + PyObject *item = NULL; + while (PyIter_NextItem(iter, &item)) { + if (item == NULL) { + Py_DECREF(iter); + return NULL; + } + if (item == value) { + Py_DECREF(item); + count++; // no overflow since count <= len(mv) <= PY_SSIZE_T_MAX + continue; + } + int contained = PyObject_RichCompareBool(item, value, Py_EQ); + Py_DECREF(item); + if (contained > 0) { // more likely than 'contained < 0' + count++; // no overflow since count <= len(mv) <= PY_SSIZE_T_MAX + } + else if (contained < 0) { + Py_DECREF(iter); + return NULL; + } + } + Py_DECREF(iter); + return PyLong_FromSsize_t(count); +} + + /**************************************************************************/ /* Lookup */ /**************************************************************************/ @@ -3370,6 +3419,7 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_CAST_METHODDEF MEMORYVIEW_TOREADONLY_METHODDEF MEMORYVIEW__FROM_FLAGS_METHODDEF + MEMORYVIEW_COUNT_METHODDEF MEMORYVIEW_INDEX_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__", memory_exit, METH_VARARGS, memory_exit_doc}, From 8dbdbad6e085564ec20109fc8277fa54b3fcc2db Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Tue, 10 Dec 2024 19:22:37 +0900 Subject: [PATCH 155/427] gh-101100: Fix sphinx warnings in `whatsnew/3.0.rst` (#127662) --- Doc/library/xmlrpc.rst | 3 ++ Doc/tools/.nitignore | 1 - Doc/whatsnew/3.0.rst | 112 ++++++++++++++++++++--------------------- 3 files changed, 59 insertions(+), 57 deletions(-) diff --git a/Doc/library/xmlrpc.rst b/Doc/library/xmlrpc.rst index 5f0a2cf68d01f9..a93d08f78cfba7 100644 --- a/Doc/library/xmlrpc.rst +++ b/Doc/library/xmlrpc.rst @@ -1,6 +1,9 @@ :mod:`!xmlrpc` --- XMLRPC server and client modules =================================================== +.. module:: xmlrpc + :synopsis: Server and client modules implementing XML-RPC. + XML-RPC is a Remote Procedure Call method that uses XML passed via HTTP as a transport. With it, a client can call methods with parameters on a remote server (the server is named by a URI) and get back structured data. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 39d1f5975e331c..7d50aec56a9bf7 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -76,7 +76,6 @@ Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst Doc/whatsnew/2.7.rst -Doc/whatsnew/3.0.rst Doc/whatsnew/3.3.rst Doc/whatsnew/3.4.rst Doc/whatsnew/3.5.rst diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index d97f5fdd9eaa4a..6e1fda22ed203a 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -150,8 +150,8 @@ Some well-known APIs no longer return lists: sorted(d)`` instead (this works in Python 2.5 too and is just as efficient). -* Also, the :meth:`dict.iterkeys`, :meth:`dict.iteritems` and - :meth:`dict.itervalues` methods are no longer supported. +* Also, the :meth:`!dict.iterkeys`, :meth:`!dict.iteritems` and + :meth:`!dict.itervalues` methods are no longer supported. * :func:`map` and :func:`filter` return iterators. If you really need a list and the input sequences are all of equal length, a quick @@ -170,7 +170,7 @@ Some well-known APIs no longer return lists: :func:`itertools.zip_longest`, e.g. ``map(func, *sequences)`` becomes ``list(map(func, itertools.zip_longest(*sequences)))``. -* :func:`range` now behaves like :func:`xrange` used to behave, except +* :func:`range` now behaves like :func:`!xrange` used to behave, except it works with values of arbitrary size. The latter no longer exists. @@ -192,33 +192,33 @@ Python 3.0 has simplified the rules for ordering comparisons: operators: objects of different incomparable types always compare unequal to each other. -* :meth:`builtin.sorted` and :meth:`list.sort` no longer accept the +* :meth:`sorted` and :meth:`list.sort` no longer accept the *cmp* argument providing a comparison function. Use the *key* argument instead. N.B. the *key* and *reverse* arguments are now "keyword-only". -* The :func:`cmp` function should be treated as gone, and the :meth:`__cmp__` - special method is no longer supported. Use :meth:`__lt__` for sorting, - :meth:`__eq__` with :meth:`__hash__`, and other rich comparisons as needed. - (If you really need the :func:`cmp` functionality, you could use the +* The :func:`!cmp` function should be treated as gone, and the :meth:`!__cmp__` + special method is no longer supported. Use :meth:`~object.__lt__` for sorting, + :meth:`~object.__eq__` with :meth:`~object.__hash__`, and other rich comparisons as needed. + (If you really need the :func:`!cmp` functionality, you could use the expression ``(a > b) - (a < b)`` as the equivalent for ``cmp(a, b)``.) Integers -------- -* :pep:`237`: Essentially, :class:`long` renamed to :class:`int`. +* :pep:`237`: Essentially, :class:`!long` renamed to :class:`int`. That is, there is only one built-in integral type, named - :class:`int`; but it behaves mostly like the old :class:`long` type. + :class:`int`; but it behaves mostly like the old :class:`!long` type. * :pep:`238`: An expression like ``1/2`` returns a float. Use ``1//2`` to get the truncating behavior. (The latter syntax has existed for years, at least since Python 2.2.) -* The :data:`sys.maxint` constant was removed, since there is no +* The :data:`!sys.maxint` constant was removed, since there is no longer a limit to the value of integers. However, :data:`sys.maxsize` can be used as an integer larger than any practical list or string index. It conforms to the implementation's "natural" integer size - and is typically the same as :data:`sys.maxint` in previous releases + and is typically the same as :data:`!sys.maxint` in previous releases on the same platform (assuming the same build options). * The :func:`repr` of a long integer doesn't include the trailing ``L`` @@ -251,7 +251,7 @@ changed. that uses Unicode, encodings or binary data most likely has to change. The change is for the better, as in the 2.x world there were numerous bugs having to do with mixing encoded and unencoded - text. To be prepared in Python 2.x, start using :class:`unicode` + text. To be prepared in Python 2.x, start using :class:`!unicode` for all unencoded text, and :class:`str` for binary or encoded data only. Then the ``2to3`` tool will do most of the work for you. @@ -269,7 +269,7 @@ changed. separate *mutable* type to hold buffered binary data, :class:`bytearray`. Nearly all APIs that accept :class:`bytes` also accept :class:`bytearray`. The mutable API is based on - :class:`collections.MutableSequence`. + :class:`collections.MutableSequence `. * All backslashes in raw string literals are interpreted literally. This means that ``'\U'`` and ``'\u'`` escapes in raw strings are not @@ -278,11 +278,11 @@ changed. single "euro" character. (Of course, this change only affects raw string literals; the euro character is ``'\u20ac'`` in Python 3.0.) -* The built-in :class:`basestring` abstract type was removed. Use +* The built-in :class:`!basestring` abstract type was removed. Use :class:`str` instead. The :class:`str` and :class:`bytes` types don't have functionality enough in common to warrant a shared base class. The ``2to3`` tool (see below) replaces every occurrence of - :class:`basestring` with :class:`str`. + :class:`!basestring` with :class:`str`. * Files opened as text files (still the default mode for :func:`open`) always use an encoding to map between strings (in memory) and bytes @@ -428,7 +428,7 @@ Changed Syntax class C(metaclass=M): ... - The module-global :data:`__metaclass__` variable is no longer + The module-global :data:`!__metaclass__` variable is no longer supported. (It was a crutch to make it easier to default to new-style classes without deriving every class from :class:`object`.) @@ -522,19 +522,19 @@ consulted for longer descriptions. *encoding*, *errors*, *newline* and *closefd*. Also note that an invalid *mode* argument now raises :exc:`ValueError`, not :exc:`IOError`. The binary file object underlying a text file - object can be accessed as :attr:`f.buffer` (but beware that the + object can be accessed as :attr:`!f.buffer` (but beware that the text object maintains a buffer of itself in order to speed up the encoding and decoding operations). -* :ref:`pep-3118`. The old builtin :func:`buffer` is now really gone; +* :ref:`pep-3118`. The old builtin :func:`!buffer` is now really gone; the new builtin :func:`memoryview` provides (mostly) similar functionality. * :ref:`pep-3119`. The :mod:`abc` module and the ABCs defined in the :mod:`collections` module plays a somewhat more prominent role in the language now, and built-in collection types like :class:`dict` - and :class:`list` conform to the :class:`collections.MutableMapping` - and :class:`collections.MutableSequence` ABCs, respectively. + and :class:`list` conform to the :class:`collections.MutableMapping ` + and :class:`collections.MutableSequence ` ABCs, respectively. * :ref:`pep-3127`. As mentioned above, the new octal literal notation is the only one supported, and binary literals have been @@ -612,7 +612,7 @@ review: :mod:`!CGIHTTPServer`, :mod:`!SimpleHTTPServer`, :mod:`!Cookie`, :mod:`!cookielib`). - * :mod:`tkinter` (all :mod:`Tkinter`-related modules except + * :mod:`tkinter` (all ``Tkinter``-related modules except :mod:`turtle`). The target audience of :mod:`turtle` doesn't really care about :mod:`tkinter`. Also note that as of Python 2.6, the functionality of :mod:`turtle` has been greatly enhanced. @@ -628,47 +628,47 @@ Some other changes to standard library modules, not covered by * Killed :mod:`!sets`. Use the built-in :func:`set` class. -* Cleanup of the :mod:`sys` module: removed :func:`sys.exitfunc`, - :func:`sys.exc_clear`, :data:`sys.exc_type`, :data:`sys.exc_value`, - :data:`sys.exc_traceback`. (Note that :data:`sys.last_type` +* Cleanup of the :mod:`sys` module: removed :func:`!sys.exitfunc`, + :func:`!sys.exc_clear`, :data:`!sys.exc_type`, :data:`!sys.exc_value`, + :data:`!sys.exc_traceback`. (Note that :data:`sys.last_type` etc. remain.) -* Cleanup of the :class:`array.array` type: the :meth:`read` and - :meth:`write` methods are gone; use :meth:`fromfile` and - :meth:`tofile` instead. Also, the ``'c'`` typecode for array is +* Cleanup of the :class:`array.array` type: the :meth:`!read` and + :meth:`!write` methods are gone; use :meth:`~array.array.fromfile` and + :meth:`~array.array.tofile` instead. Also, the ``'c'`` typecode for array is gone -- use either ``'b'`` for bytes or ``'u'`` for Unicode characters. * Cleanup of the :mod:`operator` module: removed - :func:`sequenceIncludes` and :func:`isCallable`. + :func:`!sequenceIncludes` and :func:`!isCallable`. * Cleanup of the :mod:`!thread` module: :func:`!acquire_lock` and :func:`!release_lock` are gone; use :meth:`~threading.Lock.acquire` and :meth:`~threading.Lock.release` instead. -* Cleanup of the :mod:`random` module: removed the :func:`jumpahead` API. +* Cleanup of the :mod:`random` module: removed the :func:`!jumpahead` API. * The :mod:`!new` module is gone. -* The functions :func:`os.tmpnam`, :func:`os.tempnam` and - :func:`os.tmpfile` have been removed in favor of the :mod:`tempfile` +* The functions :func:`!os.tmpnam`, :func:`!os.tempnam` and + :func:`!os.tmpfile` have been removed in favor of the :mod:`tempfile` module. * The :mod:`tokenize` module has been changed to work with bytes. The main entry point is now :func:`tokenize.tokenize`, instead of generate_tokens. -* :data:`string.letters` and its friends (:data:`string.lowercase` and - :data:`string.uppercase`) are gone. Use +* :data:`!string.letters` and its friends (:data:`!string.lowercase` and + :data:`!string.uppercase`) are gone. Use :data:`string.ascii_letters` etc. instead. (The reason for the - removal is that :data:`string.letters` and friends had + removal is that :data:`!string.letters` and friends had locale-specific behavior, which is a bad idea for such attractively named global "constants".) -* Renamed module :mod:`__builtin__` to :mod:`builtins` (removing the - underscores, adding an 's'). The :data:`__builtins__` variable +* Renamed module :mod:`!__builtin__` to :mod:`builtins` (removing the + underscores, adding an 's'). The :data:`!__builtins__` variable found in most global namespaces is unchanged. To modify a builtin, - you should use :mod:`builtins`, not :data:`__builtins__`! + you should use :mod:`builtins`, not :data:`!__builtins__`! :pep:`3101`: A New Approach To String Formatting @@ -702,9 +702,9 @@ new powerful features added: idiom for handling all exceptions except for this latter category is to use :keyword:`except` :exc:`Exception`. -* :exc:`StandardError` was removed. +* :exc:`!StandardError` was removed. -* Exceptions no longer behave as sequences. Use the :attr:`args` +* Exceptions no longer behave as sequences. Use the :attr:`~BaseException.args` attribute instead. * :pep:`3109`: Raising exceptions. You must now use :samp:`raise @@ -765,20 +765,20 @@ Operators And Special Methods When referencing a method as a class attribute, you now get a plain function object. -* :meth:`__getslice__`, :meth:`__setslice__` and :meth:`__delslice__` +* :meth:`!__getslice__`, :meth:`!__setslice__` and :meth:`!__delslice__` were killed. The syntax ``a[i:j]`` now translates to - ``a.__getitem__(slice(i, j))`` (or :meth:`__setitem__` or - :meth:`__delitem__`, when used as an assignment or deletion target, + ``a.__getitem__(slice(i, j))`` (or :meth:`~object.__setitem__` or + :meth:`~object.__delitem__`, when used as an assignment or deletion target, respectively). * :pep:`3114`: the standard :meth:`next` method has been renamed to :meth:`~iterator.__next__`. -* The :meth:`__oct__` and :meth:`__hex__` special methods are removed - -- :func:`oct` and :func:`hex` use :meth:`__index__` now to convert +* The :meth:`!__oct__` and :meth:`!__hex__` special methods are removed + -- :func:`oct` and :func:`hex` use :meth:`~object.__index__` now to convert the argument to an integer. -* Removed support for :attr:`__members__` and :attr:`__methods__`. +* Removed support for :attr:`!__members__` and :attr:`!__methods__`. * The function attributes named :attr:`!func_X` have been renamed to use the :attr:`!__X__` form, freeing up these names in the function @@ -802,7 +802,7 @@ Builtins instance will automatically be chosen. With arguments, the behavior of :func:`super` is unchanged. -* :pep:`3111`: :func:`raw_input` was renamed to :func:`input`. That +* :pep:`3111`: :func:`!raw_input` was renamed to :func:`input`. That is, the new :func:`input` function reads a line from :data:`sys.stdin` and returns it with the trailing newline stripped. It raises :exc:`EOFError` if the input is terminated prematurely. @@ -820,31 +820,31 @@ Builtins argument and a value of the same type as ``x`` when called with two arguments. -* Moved :func:`intern` to :func:`sys.intern`. +* Moved :func:`!intern` to :func:`sys.intern`. -* Removed: :func:`apply`. Instead of ``apply(f, args)`` use +* Removed: :func:`!apply`. Instead of ``apply(f, args)`` use ``f(*args)``. * Removed :func:`callable`. Instead of ``callable(f)`` you can use - ``isinstance(f, collections.Callable)``. The :func:`operator.isCallable` + ``isinstance(f, collections.Callable)``. The :func:`!operator.isCallable` function is also gone. -* Removed :func:`coerce`. This function no longer serves a purpose +* Removed :func:`!coerce`. This function no longer serves a purpose now that classic classes are gone. -* Removed :func:`execfile`. Instead of ``execfile(fn)`` use +* Removed :func:`!execfile`. Instead of ``execfile(fn)`` use ``exec(open(fn).read())``. -* Removed the :class:`file` type. Use :func:`open`. There are now several +* Removed the :class:`!file` type. Use :func:`open`. There are now several different kinds of streams that open can return in the :mod:`io` module. -* Removed :func:`reduce`. Use :func:`functools.reduce` if you really +* Removed :func:`!reduce`. Use :func:`functools.reduce` if you really need it; however, 99 percent of the time an explicit :keyword:`for` loop is more readable. -* Removed :func:`reload`. Use :func:`!imp.reload`. +* Removed :func:`!reload`. Use :func:`!imp.reload`. -* Removed. :meth:`dict.has_key` -- use the :keyword:`in` operator +* Removed. :meth:`!dict.has_key` -- use the :keyword:`in` operator instead. .. ====================================================================== From ae31df354d02e12bf656954c5c72380d96c1dc0e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:51:12 +0200 Subject: [PATCH 156/427] Add zizmor to pre-commit and fix most findings (#127749) Co-authored-by: Alex Waygood --- .github/workflows/build.yml | 11 ++++++++++- .github/workflows/documentation-links.yml | 6 +++--- .github/workflows/jit.yml | 6 ++++++ .github/workflows/lint.yml | 2 ++ .github/workflows/mypy.yml | 2 ++ .github/workflows/require-pr-label.yml | 10 ++++++---- .github/workflows/reusable-change-detection.yml | 2 ++ .github/workflows/reusable-docs.yml | 14 ++++++++++---- .github/workflows/reusable-macos.yml | 2 ++ .github/workflows/reusable-tsan.yml | 9 +++++++-- .github/workflows/reusable-ubuntu.yml | 4 +++- .github/workflows/reusable-wasi.yml | 12 +++++++----- .github/workflows/reusable-windows-msi.yml | 5 ++++- .github/workflows/reusable-windows.yml | 10 ++++++++-- .github/workflows/stale.yml | 5 ++--- .github/workflows/verify-ensurepip-wheels.yml | 2 ++ .github/zizmor.yml | 6 ++++++ .pre-commit-config.yaml | 9 +++++++-- 18 files changed, 89 insertions(+), 28 deletions(-) create mode 100644 .github/zizmor.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9b2f19fd6bcf54..8787402ccc4423 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,6 +58,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 1 + persist-credentials: false - name: Runner image version run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Check Autoconf and aclocal versions @@ -94,6 +95,8 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3.x' @@ -268,6 +271,8 @@ jobs: LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache @@ -328,6 +333,8 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -411,7 +418,7 @@ jobs: # # (GH-104097) test_sysconfig is skipped because it has tests that are # failing when executed from inside a virtual environment. - ${{ env.VENV_PYTHON }} -m test \ + "${VENV_PYTHON}" -m test \ -W \ -o \ -j4 \ @@ -446,6 +453,8 @@ jobs: ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index 43a7afec73884e..fdb4b9aa29a7c8 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -10,9 +10,6 @@ on: - 'Doc/**' - '.github/workflows/doc.yml' -permissions: - pull-requests: write - concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -20,6 +17,9 @@ concurrency: jobs: documentation-links: runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: - uses: readthedocs/actions/preview@v1 with: diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index ee30cf5786d55b..9b84998a55666d 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -32,6 +32,8 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Build tier two interpreter run: | ./configure --enable-experimental-jit=interpreter --with-pydebug @@ -85,6 +87,8 @@ jobs: runner: ${{ github.repository_owner == 'python' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3.11' @@ -138,6 +142,8 @@ jobs: - 19 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3.11' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ccde03f91983df..d74ce8fcc256dc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,6 +20,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e5b05302b5ac27..5dfa8d7bcafd78 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -51,6 +51,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.13" diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index bbedd22cc6d189..0a6277c779ff67 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,15 +4,14 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] -permissions: - issues: write - pull-requests: write - jobs: label-dnm: name: DO-NOT-MERGE if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write timeout-minutes: 10 steps: @@ -28,6 +27,9 @@ jobs: name: Unresolved review if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write timeout-minutes: 10 steps: diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 1a6fd33186840c..964bd87e815f42 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -61,6 +61,8 @@ jobs: - run: >- echo '${{ github.event_name }}' - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Check for source changes id: check run: | diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 39a97392e898aa..3962d12403919a 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -22,12 +22,14 @@ jobs: env: branch_base: 'origin/${{ github.event.pull_request.base.ref }}' branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + commits: ${{ github.event.pull_request.commits }} refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' uses: actions/checkout@v4 with: + persist-credentials: false ref: >- ${{ github.event_name == 'pull_request' @@ -39,15 +41,15 @@ jobs: if: github.event_name == 'pull_request' run: | # Fetch enough history to find a common ancestor commit (aka merge-base): - git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \ + git fetch origin "${refspec_pr}" --depth=$(( commits + 1 )) \ --no-tags --prune --no-recurse-submodules # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): - COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 ${{ env.branch_pr }} ) + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 "${branch_pr}" ) DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) # Get all commits since that commit date from the base branch (eg: master or main): - git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ + git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' uses: actions/setup-python@v5 @@ -69,7 +71,7 @@ jobs: if: github.event_name == 'pull_request' run: | python Doc/tools/check-warnings.py \ - --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ + --annotate-diff "${branch_base}" "${branch_pr}" \ --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit @@ -81,6 +83,8 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: 'Set up Python' uses: actions/setup-python@v5 with: @@ -99,6 +103,8 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/cache@v4 with: path: ~/.cache/pip diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 4c3dd10194f8cb..36ae3e27207e37 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -29,6 +29,8 @@ jobs: runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 7a4d81f0bdcad1..b5144ca3e9efc4 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -23,8 +23,13 @@ jobs: name: 'Thread sanitizer' runs-on: ubuntu-24.04 timeout-minutes: 60 + env: + OPTIONS: ${{ inputs.options }} + SUPPRESSIONS_PATH: ${{ inputs.suppressions_path }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache @@ -47,7 +52,7 @@ jobs: sudo sysctl -w vm.mmap_rnd_bits=28 - name: TSAN Option Setup run: | - echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }} handle_segv=0" >> "$GITHUB_ENV" + echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${SUPPRESSIONS_PATH} handle_segv=0" >> "$GITHUB_ENV" echo "CC=clang" >> "$GITHUB_ENV" echo "CXX=clang++" >> "$GITHUB_ENV" - name: Add ccache to PATH @@ -59,7 +64,7 @@ jobs: save: ${{ github.event_name == 'push' }} max-size: "200M" - name: Configure CPython - run: ${{ inputs.options }} + run: "${OPTIONS}" - name: Build CPython run: make -j4 - name: Display build info diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 2869202c7910c9..46c542940c8483 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -28,6 +28,8 @@ jobs: TERM: linux steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -94,7 +96,7 @@ jobs: if: ${{ !inputs.free-threading }} run: >- python Tools/build/check_warnings.py - --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt + --compiler-output-file-path="${CPYTHON_BUILDDIR}/compiler_output_ubuntu.txt" --warning-ignore-file-path "${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu" --compiler-output-type=gcc --fail-on-regression diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 3f96c888e2dd30..4356d9c1c8795e 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -20,6 +20,8 @@ jobs: CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. - name: "Install wasmtime" uses: bytecodealliance/actions/wasmtime/setup@v1 @@ -34,9 +36,9 @@ jobs: - name: "Install WASI SDK" # Hard-coded to x64. if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' run: | - mkdir ${{ env.WASI_SDK_PATH }} && \ - curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-x86_64-linux.tar.gz | \ - tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip + mkdir "${WASI_SDK_PATH}" && \ + curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz" | \ + tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - name: "Configure ccache action" uses: hendrikmuhs/ccache-action@v1.2 with: @@ -72,6 +74,6 @@ jobs: - name: "Make host" run: python3 Tools/wasm/wasi.py make-host - name: "Display build info" - run: make --directory ${{ env.CROSS_BUILD_WASI }} pythoninfo + run: make --directory "${CROSS_BUILD_WASI}" pythoninfo - name: "Test" - run: make --directory ${{ env.CROSS_BUILD_WASI }} test + run: make --directory "${CROSS_BUILD_WASI}" test diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index abdb1a1982fef8..d0d53dba0b45d1 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -17,8 +17,11 @@ jobs: runs-on: windows-latest timeout-minutes: 60 env: + ARCH: ${{ inputs.arch }} IncludeFreethreaded: true steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Build CPython installer - run: .\Tools\msi\build.bat --doc -${{ inputs.arch }} + run: .\Tools\msi\build.bat --doc -"${ARCH}" diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 12b68d68466d62..459d2b29e5d42b 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -26,8 +26,12 @@ jobs: name: 'build and test (${{ inputs.arch }})' runs-on: ${{ inputs.os }} timeout-minutes: 60 + env: + ARCH: ${{ inputs.arch }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Register MSVC problem matcher if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" @@ -35,8 +39,9 @@ jobs: run: >- .\\PCbuild\\build.bat -e -d -v - -p ${{ inputs.arch }} + -p "${ARCH}" ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + shell: bash - name: Display build info # FIXME(diegorusso): remove the `if` if: inputs.arch != 'arm64' run: .\\python.bat -m test.pythoninfo @@ -44,6 +49,7 @@ jobs: if: inputs.arch != 'arm64' run: >- .\\PCbuild\\rt.bat - -p ${{ inputs.arch }} + -p "${ARCH}" -d -q --fast-ci ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + shell: bash diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index f97587e68cbbe4..7578189f5d4d67 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,14 +4,13 @@ on: schedule: - cron: "0 */6 * * *" -permissions: - pull-requests: write - jobs: stale: if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + pull-requests: write timeout-minutes: 10 steps: diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 83b007f1c9c2ef..463e7bf3355cc3 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -26,6 +26,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3' diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 00000000000000..eeda8d9eaaf484 --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,6 @@ +# Configuration for the zizmor static analysis tool, run via pre-commit in CI +# https://woodruffw.github.io/zizmor/configuration/ +rules: + dangerous-triggers: + ignore: + - documentation-links.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ccaf2390d99fae..107f3b255735f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.8.2 hooks: - id: ruff name: Run Ruff (lint) on Doc/ @@ -51,7 +51,7 @@ repos: types_or: [c, inc, python, rst] - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.4 + rev: 0.30.0 hooks: - id: check-dependabot - id: check-github-workflows @@ -61,6 +61,11 @@ repos: hooks: - id: actionlint + - repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v0.8.0 + hooks: + - id: zizmor + - repo: https://github.com/sphinx-contrib/sphinx-lint rev: v1.0.0 hooks: From 690fe077f6b1bf50e9d62078b22c334775efb3af Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 10 Dec 2024 11:53:56 +0100 Subject: [PATCH 157/427] gh-126491: Revert "GH-126491: Lower heap size limit with faster marking (GH-127519)" (GH-127770) Revert "GH-126491: Lower heap size limit with faster marking (GH-127519)" This reverts commit 023b7d2141467017abc27de864f3f44677768cb3, which introduced a refleak. --- InternalDocs/garbage_collector.md | 51 +---- Lib/test/test_gc.py | 14 +- Objects/dictobject.c | 4 +- Objects/genobject.c | 69 ++++++- Objects/typeobject.c | 13 -- Python/gc.c | 301 ++++++++++++++++-------------- 6 files changed, 243 insertions(+), 209 deletions(-) diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 394e4ef075f55e..e4cb9e45c9e96a 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -199,22 +199,22 @@ unreachable: ```pycon >>> import gc ->>> +>>> >>> class Link: ... def __init__(self, next_link=None): ... self.next_link = next_link -... +... >>> link_3 = Link() >>> link_2 = Link(link_3) >>> link_1 = Link(link_2) >>> link_3.next_link = link_1 >>> A = link_1 >>> del link_1, link_2, link_3 ->>> +>>> >>> link_4 = Link() >>> link_4.next_link = link_4 >>> del link_4 ->>> +>>> >>> # Collect the unreachable Link object (and its .__dict__ dict). >>> gc.collect() 2 @@ -459,11 +459,11 @@ specifically in a generation by calling `gc.collect(generation=NUM)`. >>> # Create a reference cycle. >>> x = MyObj() >>> x.self = x ->>> +>>> >>> # Initially the object is in the young generation. >>> gc.get_objects(generation=0) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ->>> +>>> >>> # After a collection of the youngest generation the object >>> # moves to the old generation. >>> gc.collect(generation=0) @@ -515,45 +515,6 @@ increment. All objects directly referred to from those stack frames are added to the working set. Then the above algorithm is repeated, starting from step 2. -Determining how much work to do -------------------------------- - -We need to do a certain amount of work to ensure that garbage is collected, -but doing too much work slows down execution. - -To work out how much work we need to do, consider a heap with `L` live objects -and `G0` garbage objects at the start of a full scavenge and `G1` garbage objects -at the end of the scavenge. We don't want the amount of garbage to grow, `G1 ≤ G0`, and -we don't want too much garbage (say 1/3 of the heap maximum), `G0 ≤ L/2`. -For each full scavenge we must visit all objects, `T == L + G0 + G1`, during which -`G1` garbage objects are created. - -The number of new objects created `N` must be at least the new garbage created, `N ≥ G1`, -assuming that the number of live objects remains roughly constant. -If we set `T == 4*N` we get `T > 4*G1` and `T = L + G0 + G1` => `L + G0 > 3G1` -For a steady state heap (`G0 == G1`) we get `L > 2G0` and the desired garbage ratio. - -In other words, to keep the garbage fraction to 1/3 or less we need to visit -4 times as many objects as are newly created. - -We can do better than this though. Not all new objects will be garbage. -Consider the heap at the end of the scavenge with `L1` live objects and `G1` -garbage. Also, note that `T == M + I` where `M` is the number of objects marked -as reachable and `I` is the number of objects visited in increments. -Everything in `M` is live, so `I ≥ G0` and in practice `I` is closer to `G0 + G1`. - -If we choose the amount of work done such that `2*M + I == 6N` then we can do -less work in most cases, but are still guaranteed to keep up. -Since `I ≳ G0 + G1` (not strictly true, but close enough) -`T == M + I == (6N + I)/2` and `(6N + I)/2 ≳ 4G`, so we can keep up. - -The reason that this improves performance is that `M` is usually much larger -than `I`. If `M == 10I`, then `T ≅ 3N`. - -Finally, instead of using a fixed multiple of 8, we gradually increase it as the -heap grows. This avoids wasting work for small heaps and during startup. - - Optimization: reusing fields to save memory =========================================== diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index baf8e95dffdfce..b5140057a69d36 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1161,19 +1161,27 @@ def make_ll(depth): return head head = make_ll(1000) + count = 1000 + + # There will be some objects we aren't counting, + # e.g. the gc stats dicts. This test checks + # that the counts don't grow, so we try to + # correct for the uncounted objects + # This is just an estimate. + CORRECTION = 20 enabled = gc.isenabled() gc.enable() olds = [] initial_heap_size = _testinternalcapi.get_tracked_heap_size() - iterations = max(20_000, initial_heap_size) - for i in range(iterations): + for i in range(20_000): newhead = make_ll(20) + count += 20 newhead.surprise = head olds.append(newhead) if len(olds) == 20: new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size - self.assertLess(new_objects, initial_heap_size/2, f"Heap growing. Reached limit after {i} iterations") + self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations") del olds[:] if not enabled: gc.disable() diff --git a/Objects/dictobject.c b/Objects/dictobject.c index de518b8dc5024b..1c9f86438dadc3 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -7064,7 +7064,9 @@ int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) { PyTypeObject *tp = Py_TYPE(obj); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + return 0; + } if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictValues *values = _PyObject_InlineValues(obj); if (values->valid) { diff --git a/Objects/genobject.c b/Objects/genobject.c index 33679afecb420f..e87f199c2504ba 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -882,7 +882,25 @@ PyTypeObject PyGen_Type = { gen_methods, /* tp_methods */ gen_memberlist, /* tp_members */ gen_getsetlist, /* tp_getset */ - .tp_finalize = _PyGen_Finalize, + 0, /* tp_base */ + 0, /* tp_dict */ + + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + _PyGen_Finalize, /* tp_finalize */ }; static PyObject * @@ -1224,7 +1242,24 @@ PyTypeObject PyCoro_Type = { coro_methods, /* tp_methods */ coro_memberlist, /* tp_members */ coro_getsetlist, /* tp_getset */ - .tp_finalize = _PyGen_Finalize, + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + _PyGen_Finalize, /* tp_finalize */ }; static void @@ -1429,6 +1464,7 @@ typedef struct _PyAsyncGenWrappedValue { (assert(_PyAsyncGenWrappedValue_CheckExact(op)), \ _Py_CAST(_PyAsyncGenWrappedValue*, (op))) + static int async_gen_traverse(PyObject *self, visitproc visit, void *arg) { @@ -1637,7 +1673,24 @@ PyTypeObject PyAsyncGen_Type = { async_gen_methods, /* tp_methods */ async_gen_memberlist, /* tp_members */ async_gen_getsetlist, /* tp_getset */ - .tp_finalize = _PyGen_Finalize, + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + _PyGen_Finalize, /* tp_finalize */ }; @@ -1882,6 +1935,16 @@ PyTypeObject _PyAsyncGenASend_Type = { PyObject_SelfIter, /* tp_iter */ async_gen_asend_iternext, /* tp_iternext */ async_gen_asend_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ .tp_finalize = async_gen_asend_finalize, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index cc95b9857e3f2d..2068d6aa9be52b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2355,16 +2355,6 @@ subtype_traverse(PyObject *self, visitproc visit, void *arg) return 0; } - -static int -plain_object_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyTypeObject *type = Py_TYPE(self); - assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); - Py_VISIT(type); - return PyObject_VisitManagedDict(self, visit, arg); -} - static void clear_slots(PyTypeObject *type, PyObject *self) { @@ -4157,9 +4147,6 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) assert((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); type->tp_flags |= Py_TPFLAGS_MANAGED_DICT; type->tp_dictoffset = -1; - if (type->tp_basicsize == sizeof(PyObject)) { - type->tp_traverse = plain_object_traverse; - } } type->tp_basicsize = slotoffset; diff --git a/Python/gc.c b/Python/gc.c index fd29a48518e71b..5b9588c8741b97 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1277,13 +1277,18 @@ gc_list_set_space(PyGC_Head *list, int space) * space faster than objects are added to the old space. * * Each young or incremental collection adds a number of - * new objects (N) to the heap, and incremental collectors - * scan I objects from the old space. - * I > N must be true. We also want I > N * K to be where - * K > 2. Higher values of K mean that the old space is + * objects, S (for survivors) to the old space, and + * incremental collectors scan I objects from the old space. + * I > S must be true. We also want I > S * N to be where + * N > 1. Higher values of N mean that the old space is * scanned more rapidly. + * The default incremental threshold of 10 translates to + * N == 1.4 (1 + 4/threshold) */ -#define SCAN_RATE_DIVISOR 5 + +/* Divide by 10, so that the default incremental threshold of 10 + * scans objects at 1% of the heap size */ +#define SCAN_RATE_DIVISOR 10 static void add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats) @@ -1325,76 +1330,69 @@ gc_collect_young(PyThreadState *tstate, validate_spaces(gcstate); } -typedef struct work_stack { - PyGC_Head *top; - int visited_space; -} WorkStack; - -/* Remove gc from the list it is currently in and push it to the stack */ -static inline void -push_to_stack(PyGC_Head *gc, WorkStack *stack) -{ - PyGC_Head *prev = GC_PREV(gc); - PyGC_Head *next = GC_NEXT(gc); - _PyGCHead_SET_NEXT(prev, next); - _PyGCHead_SET_PREV(next, prev); - _PyGCHead_SET_PREV(gc, stack->top); - stack->top = gc; -} - -static inline PyGC_Head * -pop_from_stack(WorkStack *stack) +#ifndef NDEBUG +static inline int +IS_IN_VISITED(PyGC_Head *gc, int visited_space) { - PyGC_Head *gc = stack->top; - stack->top = _PyGCHead_PREV(gc); - return gc; + assert(visited_space == 0 || other_space(visited_space) == 0); + return gc_old_space(gc) == visited_space; } +#endif -/* append list `from` to `stack`; `from` becomes an empty list */ -static void -move_list_to_stack(PyGC_Head *from, WorkStack *stack) -{ - if (!gc_list_is_empty(from)) { - PyGC_Head *from_head = GC_NEXT(from); - PyGC_Head *from_tail = GC_PREV(from); - _PyGCHead_SET_PREV(from_head, stack->top); - stack->top = from_tail; - gc_list_init(from); - } -} +struct container_and_flag { + PyGC_Head *container; + int visited_space; + intptr_t size; +}; -static inline void -move_to_stack(PyObject *op, WorkStack *stack, int visited_space) +/* A traversal callback for adding to container) */ +static int +visit_add_to_container(PyObject *op, void *arg) { - assert(op != NULL); - if (_PyObject_IS_GC(op)) { + OBJECT_STAT_INC(object_visits); + struct container_and_flag *cf = (struct container_and_flag *)arg; + int visited = cf->visited_space; + assert(visited == get_gc_state()->visited_space); + if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op) && - gc_old_space(gc) != visited_space) { - assert(!_Py_IsImmortal(op)); + gc_old_space(gc) != visited) { gc_flip_old_space(gc); - push_to_stack(gc, stack); + gc_list_move(gc, cf->container); + cf->size++; } } + return 0; } -static void -move_unvisited(PyObject *op, WorkStack *stack, int visited_space) -{ - move_to_stack(op, stack, visited_space); -} - -#define MOVE_UNVISITED(O, T, V) if ((O) != NULL) move_unvisited((O), (T), (V)) - -/* A traversal callback for adding to container */ -static int -visit_add_to_container(PyObject *op, void *arg) -{ - OBJECT_STAT_INC(object_visits); - WorkStack *stack = (WorkStack *)arg; - assert(stack->visited_space == get_gc_state()->visited_space); - move_to_stack(op, stack, stack->visited_space); - return 0; +static intptr_t +expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate) +{ + struct container_and_flag arg = { + .container = container, + .visited_space = gcstate->visited_space, + .size = 0 + }; + assert(GC_NEXT(gc) == container); + while (gc != container) { + /* Survivors will be moved to visited space, so they should + * have been marked as visited */ + assert(IS_IN_VISITED(gc, gcstate->visited_space)); + PyObject *op = FROM_GC(gc); + assert(_PyObject_GC_IS_TRACKED(op)); + if (_Py_IsImmortal(op)) { + PyGC_Head *next = GC_NEXT(gc); + gc_list_move(gc, &get_gc_state()->permanent_generation.head); + gc = next; + continue; + } + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_add_to_container, + &arg); + gc = GC_NEXT(gc); + } + return arg.size; } /* Do bookkeeping for a completed GC cycle */ @@ -1422,62 +1420,54 @@ completed_scavenge(GCState *gcstate) gc_list_set_space(&gcstate->old[not_visited].head, not_visited); } assert(gc_list_is_empty(&gcstate->old[visited].head)); + gcstate->work_to_do = 0; gcstate->phase = GC_PHASE_MARK; } -static void -frame_move_unvisited(_PyInterpreterFrame *frame, WorkStack *stack, int visited_space) -{ - _PyStackRef *locals = frame->localsplus; - _PyStackRef *sp = frame->stackpointer; - if (frame->f_locals != NULL) { - move_unvisited(frame->f_locals, stack, visited_space); - } - PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - move_unvisited(func, stack, visited_space); - while (sp > locals) { - sp--; - _PyStackRef ref = *sp; - if (!PyStackRef_IsNull(ref)) { - PyObject *op = PyStackRef_AsPyObjectBorrow(ref); - if (!_Py_IsImmortal(op)) { - move_unvisited(op, stack, visited_space); - } +static intptr_t +move_to_reachable(PyObject *op, PyGC_Head *reachable, int visited_space) +{ + if (op != NULL && !_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited_space) { + gc_flip_old_space(gc); + gc_list_move(gc, reachable); + return 1; } } + return 0; } -static Py_ssize_t -move_all_transitively_reachable(WorkStack *stack, PyGC_Head *visited, int visited_space) +static intptr_t +mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space) { // Transitively traverse all objects from reachable, until empty - Py_ssize_t objects_marked = 0; - while (stack->top != NULL) { - PyGC_Head *gc = pop_from_stack(stack); + struct container_and_flag arg = { + .container = reachable, + .visited_space = visited_space, + .size = 0 + }; + while (!gc_list_is_empty(reachable)) { + PyGC_Head *gc = _PyGCHead_NEXT(reachable); assert(gc_old_space(gc) == visited_space); - gc_list_append(gc, visited); - objects_marked++; + gc_list_move(gc, visited); PyObject *op = FROM_GC(gc); - assert(PyObject_IS_GC(op)); - assert(_PyObject_GC_IS_TRACKED(op)); - if (_Py_IsImmortal(op)) { - _PyObject_GC_UNTRACK(op); - } - else { - traverseproc traverse = Py_TYPE(op)->tp_traverse; - (void) traverse(op, visit_add_to_container, stack); - } + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_add_to_container, + &arg); } gc_list_validate_space(visited, visited_space); - return objects_marked; + return arg.size; } static intptr_t mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start) { - WorkStack stack; - stack.top = NULL; - stack.visited_space = visited_space; + PyGC_Head reachable; + gc_list_init(&reachable); + Py_ssize_t objects_marked = 0; // Move all objects on stacks to reachable _PyRuntimeState *runtime = &_PyRuntime; HEAD_LOCK(runtime); @@ -1490,7 +1480,27 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b frame = frame->previous; continue; } - frame_move_unvisited(frame, &stack, visited_space); + _PyStackRef *locals = frame->localsplus; + _PyStackRef *sp = frame->stackpointer; + objects_marked += move_to_reachable(frame->f_locals, &reachable, visited_space); + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + objects_marked += move_to_reachable(func, &reachable, visited_space); + while (sp > locals) { + sp--; + if (PyStackRef_IsNull(*sp)) { + continue; + } + PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); + if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited_space) { + gc_flip_old_space(gc); + objects_marked++; + gc_list_move(gc, &reachable); + } + } + } if (!start && frame->visited) { // If this frame has already been visited, then the lower frames // will have already been visited and will not have changed @@ -1503,31 +1513,31 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b ts = PyThreadState_Next(ts); HEAD_UNLOCK(runtime); } - Py_ssize_t objects_marked = move_all_transitively_reachable(&stack, visited, visited_space); - assert(stack.top == NULL); + objects_marked += mark_all_reachable(&reachable, visited, visited_space); + assert(gc_list_is_empty(&reachable)); return objects_marked; } static intptr_t mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space) { - WorkStack stack; - stack.top = NULL; - stack.visited_space = visited_space; - MOVE_UNVISITED(interp->sysdict, &stack, visited_space); - MOVE_UNVISITED(interp->builtins, &stack, visited_space); - MOVE_UNVISITED(interp->dict, &stack, visited_space); + PyGC_Head reachable; + gc_list_init(&reachable); + Py_ssize_t objects_marked = 0; + objects_marked += move_to_reachable(interp->sysdict, &reachable, visited_space); + objects_marked += move_to_reachable(interp->builtins, &reachable, visited_space); + objects_marked += move_to_reachable(interp->dict, &reachable, visited_space); struct types_state *types = &interp->types; for (int i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { - MOVE_UNVISITED(types->builtins.initialized[i].tp_dict, &stack, visited_space); - MOVE_UNVISITED(types->builtins.initialized[i].tp_subclasses, &stack, visited_space); + objects_marked += move_to_reachable(types->builtins.initialized[i].tp_dict, &reachable, visited_space); + objects_marked += move_to_reachable(types->builtins.initialized[i].tp_subclasses, &reachable, visited_space); } for (int i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { - MOVE_UNVISITED(types->for_extensions.initialized[i].tp_dict, &stack, visited_space); - MOVE_UNVISITED(types->for_extensions.initialized[i].tp_subclasses, &stack, visited_space); + objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_dict, &reachable, visited_space); + objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_subclasses, &reachable, visited_space); } - Py_ssize_t objects_marked = move_all_transitively_reachable(&stack, visited, visited_space); - assert(stack.top == NULL); + objects_marked += mark_all_reachable(&reachable, visited, visited_space); + assert(gc_list_is_empty(&reachable)); return objects_marked; } @@ -1539,35 +1549,39 @@ mark_at_start(PyThreadState *tstate) PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space); objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true); + gcstate->work_to_do -= objects_marked; gcstate->phase = GC_PHASE_COLLECT; validate_spaces(gcstate); return objects_marked; } - -/* See InternalDocs/garbage_collector.md for more details. */ -#define MAX_HEAP_PORTION_MULTIPLIER 5 -#define MARKING_PROGRESS_MULTIPLIER 2 - static intptr_t assess_work_to_do(GCState *gcstate) { - /* The amount of work we want to do depends on two things. + /* The amount of work we want to do depends on three things. * 1. The number of new objects created - * 2. The heap size (up to a multiple of the number of new objects, to avoid quadratic effects) + * 2. The growth in heap size since the last collection + * 3. The heap size (up to the number of new objects, to avoid quadratic effects) + * + * For a steady state heap, the amount of work to do is three times the number + * of new objects added to the heap. This ensures that we stay ahead in the + * worst case of all new objects being garbage. + * + * This could be improved by tracking survival rates, but it is still a + * large improvement on the non-marking approach. */ intptr_t scale_factor = gcstate->old[0].threshold; if (scale_factor < 2) { scale_factor = 2; } intptr_t new_objects = gcstate->young.count; - intptr_t max_heap_portion = new_objects * MAX_HEAP_PORTION_MULTIPLIER; - intptr_t heap_portion = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; - if (heap_portion > max_heap_portion) { - heap_portion = max_heap_portion; + intptr_t max_heap_fraction = new_objects*3/2; + intptr_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; + if (heap_fraction > max_heap_fraction) { + heap_fraction = max_heap_fraction; } gcstate->young.count = 0; - return new_objects + heap_portion; + return new_objects + heap_fraction; } static void @@ -1580,37 +1594,36 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) if (gcstate->phase == GC_PHASE_MARK) { Py_ssize_t objects_marked = mark_at_start(tstate); GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); - gcstate->work_to_do -= objects_marked * MARKING_PROGRESS_MULTIPLIER; + gcstate->work_to_do -= objects_marked; validate_spaces(gcstate); return; } PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + PyGC_Head increment; + gc_list_init(&increment); + int scale_factor = gcstate->old[0].threshold; + if (scale_factor < 2) { + scale_factor = 2; + } intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false); GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); - gcstate->work_to_do -= objects_marked * MARKING_PROGRESS_MULTIPLIER; + gcstate->work_to_do -= objects_marked; gc_list_set_space(&gcstate->young.head, gcstate->visited_space); - PyGC_Head increment; - gc_list_init(&increment); - WorkStack working; - working.top = 0; - working.visited_space = gcstate->visited_space; - move_list_to_stack(&gcstate->young.head, &working); - Py_ssize_t increment_size = move_all_transitively_reachable(&working, &increment, gcstate->visited_space); + gc_list_merge(&gcstate->young.head, &increment); gc_list_validate_space(&increment, gcstate->visited_space); - assert(working.top == NULL); + Py_ssize_t increment_size = gc_list_size(&increment); while (increment_size < gcstate->work_to_do) { if (gc_list_is_empty(not_visited)) { break; } PyGC_Head *gc = _PyGCHead_NEXT(not_visited); - gc_set_old_space(gc, gcstate->visited_space); - push_to_stack(gc, &working); + gc_list_move(gc, &increment); + increment_size++; assert(!_Py_IsImmortal(FROM_GC(gc))); - increment_size += move_all_transitively_reachable(&working, &increment, gcstate->visited_space); - assert(working.top == NULL); + gc_set_old_space(gc, gcstate->visited_space); + increment_size += expand_region_transitively_reachable(&increment, gc, gcstate); } - assert(increment_size == gc_list_size(&increment)); GC_STAT_ADD(1, objects_not_transitively_reachable, increment_size); validate_list(&increment, collecting_clear_unreachable_clear); gc_list_validate_space(&increment, gcstate->visited_space); @@ -1619,6 +1632,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) gc_collect_region(tstate, &increment, &survivors, stats); gc_list_merge(&survivors, visited); assert(gc_list_is_empty(&increment)); + gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; gcstate->work_to_do -= increment_size; add_stats(gcstate, 1, stats); @@ -1654,7 +1668,6 @@ gc_collect_full(PyThreadState *tstate, gcstate->old[0].count = 0; gcstate->old[1].count = 0; completed_scavenge(gcstate); - gcstate->work_to_do = -gcstate->young.threshold; _PyGC_ClearAllFreeLists(tstate->interp); validate_spaces(gcstate); add_stats(gcstate, 2, stats); From f4b31edf2d9d72878dab1f66a36913b5bcc848ec Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 10 Dec 2024 11:56:24 +0100 Subject: [PATCH 158/427] gh-127257: ssl: Raise OSError for ERR_LIB_SYS (GH-127361) From the ERR_raise manpage: ERR_LIB_SYS This "library code" indicates that a system error is being reported. In this case, the reason code given to `ERR_raise()` and `ERR_raise_data()` *must* be `errno(3)`. This PR only handles ERR_LIB_SYS for the high-lever error types SSL_ERROR_SYSCALL and SSL_ERROR_SSL, i.e., not the ones where OpenSSL indicates it has some more information about the issue. --- .../2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst | 2 ++ Modules/_ssl.c | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst diff --git a/Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst b/Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst new file mode 100644 index 00000000000000..fb0380cba0b607 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst @@ -0,0 +1,2 @@ +In :mod:`ssl`, system call failures that OpenSSL reports using +``ERR_LIB_SYS`` are now raised as :exc:`OSError`. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 59c414f9ce1ceb..e7df132869fee6 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -667,6 +667,11 @@ PySSL_SetError(PySSLSocket *sslsock, const char *filename, int lineno) ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { type = state->PySSLCertVerificationErrorObject; } + if (ERR_GET_LIB(e) == ERR_LIB_SYS) { + // A system error is being reported; reason is set to errno + errno = ERR_GET_REASON(e); + return PyErr_SetFromErrno(PyExc_OSError); + } p = PY_SSL_ERROR_SYSCALL; } break; @@ -692,6 +697,11 @@ PySSL_SetError(PySSLSocket *sslsock, const char *filename, int lineno) errstr = "EOF occurred in violation of protocol"; } #endif + if (ERR_GET_LIB(e) == ERR_LIB_SYS) { + // A system error is being reported; reason is set to errno + errno = ERR_GET_REASON(e); + return PyErr_SetFromErrno(PyExc_OSError); + } break; } default: From cef0a90d8f3a94aa534593f39b4abf98165675b9 Mon Sep 17 00:00:00 2001 From: Melissa0x1f992 <70096546+Melissa0x1f992@users.noreply.github.com> Date: Tue, 10 Dec 2024 06:13:11 -0600 Subject: [PATCH 159/427] gh-126937: ctypes: fix TypeError when a field's size is >65535 bytes (GH-126938) Co-authored-by: Peter Bierma Co-authored-by: Terry Jan Reedy Co-authored-by: Petr Viktorin --- Lib/test/test_ctypes/test_struct_fields.py | 23 +++++++++++++++++++ ...-11-17-21-35-55.gh-issue-126937.qluVM0.rst | 3 +++ Modules/_ctypes/cfield.c | 10 ++++++-- Modules/_ctypes/stgdict.c | 4 ++-- 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index b5e165f3bae929..1b3e64efd410b8 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -1,4 +1,5 @@ import unittest +import sys from ctypes import Structure, Union, sizeof, c_char, c_int from ._support import (CField, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -75,6 +76,28 @@ def __init_subclass__(cls, **kwargs): 'ctypes state is not initialized'): class Subclass(BrokenStructure): ... + def test_max_field_size_gh126937(self): + # Classes for big structs should be created successfully. + # (But they most likely can't be instantiated.) + # Here we test the exact limit: the number of *bits* must fit + # in Py_ssize_t. + + class X(self.cls): + _fields_ = [('char', c_char),] + max_field_size = sys.maxsize // 8 + + class Y(self.cls): + _fields_ = [('largeField', X * max_field_size)] + class Z(self.cls): + _fields_ = [('largeField', c_char * max_field_size)] + + with self.assertRaises(ValueError): + class TooBig(self.cls): + _fields_ = [('largeField', X * (max_field_size + 1))] + with self.assertRaises(ValueError): + class TooBig(self.cls): + _fields_ = [('largeField', c_char * (max_field_size + 1))] + # __set__ and __get__ should raise a TypeError in case their self # argument is not a ctype instance. def test___set__(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst new file mode 100644 index 00000000000000..8d7da0d4107021 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst @@ -0,0 +1,3 @@ +Fix :exc:`TypeError` when a :class:`ctypes.Structure` has a field size +that doesn't fit into an unsigned 16-bit integer. +Instead, the maximum number of *bits* is :data:`sys.maxsize`. diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 3220852c8398e0..2b9e8a1a10d6f5 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -110,10 +110,16 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, goto error; } - Py_ssize_t bit_size = NUM_BITS(size); - if (bit_size) { + if (bit_size_obj != Py_None) { +#ifdef Py_DEBUG + Py_ssize_t bit_size = NUM_BITS(size); assert(bit_size > 0); assert(bit_size <= info->size * 8); + // Currently, the bit size is specified redundantly + // in NUM_BITS(size) and bit_size_obj. + // Verify that they match. + assert(PyLong_AsSsize_t(bit_size_obj) == bit_size); +#endif switch(info->ffi_type_pointer.type) { case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 5dbbe0b3285d58..5ca5b62427600d 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -292,7 +292,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (!tmp) { goto error; } - Py_ssize_t total_align = PyLong_AsInt(tmp); + Py_ssize_t total_align = PyLong_AsSsize_t(tmp); Py_DECREF(tmp); if (total_align < 0) { if (!PyErr_Occurred()) { @@ -306,7 +306,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (!tmp) { goto error; } - Py_ssize_t total_size = PyLong_AsInt(tmp); + Py_ssize_t total_size = PyLong_AsSsize_t(tmp); Py_DECREF(tmp); if (total_size < 0) { if (!PyErr_Occurred()) { From 9af96f440618304e7cc609c246e1f8c8b2d7a119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:58:17 +0100 Subject: [PATCH 160/427] gh-127563: use `dk_log2_index_bytes=3` in empty dicts (GH-127568) This fixes a UBSan failure (unaligned zero-size memcpy) in `dictobject.c`. --- Objects/dictobject.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 1c9f86438dadc3..05c93a3e448181 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -588,11 +588,14 @@ estimate_log2_keysize(Py_ssize_t n) /* This immutable, empty PyDictKeysObject is used for PyDict_Clear() * (which cannot fail and thus can do no allocation). + * + * See https://github.com/python/cpython/pull/127568#discussion_r1868070614 + * for the rationale of using dk_log2_index_bytes=3 instead of 0. */ static PyDictKeysObject empty_keys_struct = { _Py_DICT_IMMORTAL_INITIAL_REFCNT, /* dk_refcnt */ 0, /* dk_log2_size */ - 0, /* dk_log2_index_bytes */ + 3, /* dk_log2_index_bytes */ DICT_KEYS_UNICODE, /* dk_kind */ #ifdef Py_GIL_DISABLED {0}, /* dk_mutex */ From c91ccbe4ac0ec15c503521f539b3528db85871b4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 10 Dec 2024 17:33:11 +0100 Subject: [PATCH 161/427] gh-59705: Set OS thread name when Thread.name is changed (#127702) Co-authored-by: Petr Viktorin --- Doc/library/threading.rst | 12 ++++++++++++ Lib/test/test_threading.py | 19 +++++++++++++++++++ Lib/threading.py | 16 +++++++++++----- Modules/_threadmodule.c | 3 +-- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index d4b343db36efb3..f183f3f535c4cb 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -434,6 +434,18 @@ since it is impossible to detect the termination of alien threads. Multiple threads may be given the same name. The initial name is set by the constructor. + On some platforms, the thread name is set at the operating system level + when the thread starts, so that it is visible in task managers. + This name may be truncated to fit in a system-specific limit (for example, + 15 bytes on Linux or 63 bytes on macOS). + + Changes to *name* are only reflected at the OS level when the currently + running thread is renamed. (Setting the *name* attribute of a + different thread only updates the Python Thread object.) + + .. versionchanged:: 3.14 + Set the operating system thread name. + .. method:: getName() setName() diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index d05161f46f1034..3e164a12581dd1 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -2164,6 +2164,25 @@ def work(): self.assertEqual(work_name, expected, f"{len(work_name)=} and {len(expected)=}") + @unittest.skipUnless(hasattr(_thread, 'set_name'), "missing _thread.set_name") + @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") + def test_change_name(self): + # Change the name of a thread while the thread is running + + name1 = None + name2 = None + def work(): + nonlocal name1, name2 + name1 = _thread._get_name() + threading.current_thread().name = "new name" + name2 = _thread._get_name() + + thread = threading.Thread(target=work, name="name") + thread.start() + thread.join() + self.assertEqual(name1, "name") + self.assertEqual(name2, "new name") + class InterruptMainTests(unittest.TestCase): def check_interrupt_main_with_signal_handler(self, signum): diff --git a/Lib/threading.py b/Lib/threading.py index 3abd22a2aa1b72..78e591124278fc 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1026,16 +1026,20 @@ def _set_ident(self): def _set_native_id(self): self._native_id = get_native_id() + def _set_os_name(self): + if _set_name is None or not self._name: + return + try: + _set_name(self._name) + except OSError: + pass + def _bootstrap_inner(self): try: self._set_ident() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() - if _set_name is not None and self._name: - try: - _set_name(self._name) - except OSError: - pass + self._set_os_name() self._started.set() with _active_limbo_lock: _active[self._ident] = self @@ -1115,6 +1119,8 @@ def name(self): def name(self, name): assert self._initialized, "Thread.__init__() not called" self._name = str(name) + if get_ident() == self._ident: + self._set_os_name() @property def ident(self): diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 35c032fbeaa94f..75b34a8df7622c 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -2423,8 +2423,7 @@ _thread_set_name_impl(PyObject *module, PyObject *name_obj) #ifdef PYTHREAD_NAME_MAXLEN // Truncate to PYTHREAD_NAME_MAXLEN bytes + the NUL byte if needed - size_t len = PyBytes_GET_SIZE(name_encoded); - if (len > PYTHREAD_NAME_MAXLEN) { + if (PyBytes_GET_SIZE(name_encoded) > PYTHREAD_NAME_MAXLEN) { PyObject *truncated; truncated = PyBytes_FromStringAndSize(PyBytes_AS_STRING(name_encoded), PYTHREAD_NAME_MAXLEN); From 035f512046337e64a018d11fdaa3b21758625291 Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Wed, 11 Dec 2024 02:35:00 +0900 Subject: [PATCH 162/427] Docs: Fix indents in `xmlrpc.client.rst` (#127782) --- Doc/library/xmlrpc.client.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index c57f433e6efd98..971e65605841e7 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -64,11 +64,11 @@ between conformable Python objects and XML on the wire. The obsolete *use_datetime* flag is similar to *use_builtin_types* but it applies only to date/time values. -.. versionchanged:: 3.3 - The *use_builtin_types* flag was added. + .. versionchanged:: 3.3 + The *use_builtin_types* flag was added. -.. versionchanged:: 3.8 - The *headers* parameter was added. + .. versionchanged:: 3.8 + The *headers* parameter was added. Both the HTTP and HTTPS transports support the URL syntax extension for HTTP Basic Authentication: ``http://user:pass@host:port/path``. The ``user:pass`` From 51216857ca8283f5b41c8cf9874238da56da4968 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 11 Dec 2024 05:32:04 +0800 Subject: [PATCH 163/427] gh-126821: Add versionadded annotation to use_system_logger feature. (#127755) Add versionadded annotation to use_system_logger feature. --- Doc/c-api/init_config.rst | 2 ++ iOS/README.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 7497bf241fb10e..6b33d93a9f2af9 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1290,6 +1290,8 @@ PyConfig Default: ``0`` (don't use system log). + .. versionadded:: 3.13.2 + .. c:member:: int user_site_directory If non-zero, add the user site directory to :data:`sys.path`. diff --git a/iOS/README.rst b/iOS/README.rst index 9cea98cf1abbfa..13b885144932e4 100644 --- a/iOS/README.rst +++ b/iOS/README.rst @@ -286,7 +286,7 @@ This will: * Run the test suite on an "iPhone SE (3rd generation)" simulator. On success, the test suite will exit and report successful completion of the -test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 +test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 minutes to run; a couple of extra minutes is required to compile the testbed project, and then boot and prepare the iOS simulator. From 12b4f1a5a175d4dcec27631fce2883038f0917ae Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 11 Dec 2024 00:09:55 +0000 Subject: [PATCH 164/427] GH-127381: pathlib ABCs: remove `PathBase.samefile()` and rarer `is_*()` (#127709) Remove `PathBase.samefile()`, which is fairly specific to the local FS, and relies on `stat()`, which we're aiming to remove from `PathBase`. Also remove `PathBase.is_mount()`, `is_junction()`, `is_block_device()`, `is_char_device()`, `is_fifo()` and `is_socket()`. These rely on POSIX file type numbers that we're aiming to remove from the `PathBase` API. --- Lib/pathlib/_abc.py | 89 +---------------------- Lib/pathlib/_local.py | 65 ++++++++++++++++- Lib/test/test_pathlib/test_pathlib.py | 77 +++++++++++++++++++- Lib/test/test_pathlib/test_pathlib_abc.py | 81 --------------------- 4 files changed, 141 insertions(+), 171 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index f68685f21d6d79..02c6e0500617aa 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -16,7 +16,7 @@ import posixpath from errno import EINVAL from glob import _GlobberBase, _no_recurse_symlinks -from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from stat import S_ISDIR, S_ISLNK, S_ISREG from pathlib._os import copyfileobj @@ -408,26 +408,6 @@ def is_file(self, *, follow_symlinks=True): except (OSError, ValueError): return False - def is_mount(self): - """ - Check if this path is a mount point - """ - # Need to exist and be a dir - if not self.exists() or not self.is_dir(): - return False - - try: - parent_dev = self.parent.stat().st_dev - except OSError: - return False - - dev = self.stat().st_dev - if dev != parent_dev: - return True - ino = self.stat().st_ino - parent_ino = self.parent.stat().st_ino - return ino == parent_ino - def is_symlink(self): """ Whether this path is a symbolic link. @@ -437,76 +417,11 @@ def is_symlink(self): except (OSError, ValueError): return False - def is_junction(self): - """ - Whether this path is a junction. - """ - # Junctions are a Windows-only feature, not present in POSIX nor the - # majority of virtual filesystems. There is no cross-platform idiom - # to check for junctions (using stat().st_mode). - return False - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except (OSError, ValueError): - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except (OSError, ValueError): - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except (OSError, ValueError): - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except (OSError, ValueError): - return False - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). - """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self.with_segments(other_path).stat() - return (st.st_ino == other_st.st_ino and - st.st_dev == other_st.st_dev) - def _ensure_different_file(self, other_path): """ Raise OSError(EINVAL) if both paths refer to the same file. """ - try: - if not self.samefile(other_path): - return - except (OSError, ValueError): - return - err = OSError(EINVAL, "Source and target are the same file") - err.filename = str(self) - err.filename2 = str(other_path) - raise err + pass def _ensure_distinct_path(self, other_path): """ diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index f87069ce70a2de..85437ec80bfcc4 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -4,9 +4,10 @@ import os import posixpath import sys -from errno import EXDEV +from errno import EINVAL, EXDEV from glob import _StringGlobber from itertools import chain +from stat import S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from _collections_abc import Sequence try: @@ -596,6 +597,68 @@ def is_junction(self): """ return os.path.isjunction(self) + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return (st.st_ino == other_st.st_ino and + st.st_dev == other_st.st_dev) + + def _ensure_different_file(self, other_path): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + try: + if not self.samefile(other_path): + return + except (OSError, ValueError): + return + err = OSError(EINVAL, "Source and target are the same file") + err.filename = str(self) + err.filename2 = str(other_path) + raise err + def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index ce0f4748c860b1..b57ef420bfcbcd 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1786,13 +1786,31 @@ def test_lstat_nosymlink(self): st = p.stat() self.assertEqual(st, p.lstat()) - def test_is_junction(self): + def test_is_junction_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_junction()) + self.assertFalse((P / 'dirA').is_junction()) + self.assertFalse((P / 'non-existing').is_junction()) + self.assertFalse((P / 'fileA' / 'bah').is_junction()) + self.assertFalse((P / 'fileA\udfff').is_junction()) + self.assertFalse((P / 'fileA\x00').is_junction()) + + def test_is_junction_true(self): P = self.cls(self.base) with mock.patch.object(P.parser, 'isjunction'): self.assertEqual(P.is_junction(), P.parser.isjunction.return_value) P.parser.isjunction.assert_called_once_with(P) + def test_is_fifo_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_fifo()) + self.assertFalse((P / 'dirA').is_fifo()) + self.assertFalse((P / 'non-existing').is_fifo()) + self.assertFalse((P / 'fileA' / 'bah').is_fifo()) + self.assertIs((P / 'fileA\udfff').is_fifo(), False) + self.assertIs((P / 'fileA\x00').is_fifo(), False) + @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") @unittest.skipIf(sys.platform == "vxworks", "fifo requires special path on VxWorks") @@ -1808,6 +1826,15 @@ def test_is_fifo_true(self): self.assertIs(self.cls(self.base, 'myfifo\udfff').is_fifo(), False) self.assertIs(self.cls(self.base, 'myfifo\x00').is_fifo(), False) + def test_is_socket_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_socket()) + self.assertFalse((P / 'dirA').is_socket()) + self.assertFalse((P / 'non-existing').is_socket()) + self.assertFalse((P / 'fileA' / 'bah').is_socket()) + self.assertIs((P / 'fileA\udfff').is_socket(), False) + self.assertIs((P / 'fileA\x00').is_socket(), False) + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") @unittest.skipIf( is_emscripten, "Unix sockets are not implemented on Emscripten." @@ -1831,6 +1858,24 @@ def test_is_socket_true(self): self.assertIs(self.cls(self.base, 'mysock\udfff').is_socket(), False) self.assertIs(self.cls(self.base, 'mysock\x00').is_socket(), False) + def test_is_block_device_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_block_device()) + self.assertFalse((P / 'dirA').is_block_device()) + self.assertFalse((P / 'non-existing').is_block_device()) + self.assertFalse((P / 'fileA' / 'bah').is_block_device()) + self.assertIs((P / 'fileA\udfff').is_block_device(), False) + self.assertIs((P / 'fileA\x00').is_block_device(), False) + + def test_is_char_device_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_char_device()) + self.assertFalse((P / 'dirA').is_char_device()) + self.assertFalse((P / 'non-existing').is_char_device()) + self.assertFalse((P / 'fileA' / 'bah').is_char_device()) + self.assertIs((P / 'fileA\udfff').is_char_device(), False) + self.assertIs((P / 'fileA\x00').is_char_device(), False) + def test_is_char_device_true(self): # os.devnull should generally be a char device. P = self.cls(os.devnull) @@ -1842,7 +1887,14 @@ def test_is_char_device_true(self): self.assertIs(self.cls(f'{os.devnull}\udfff').is_char_device(), False) self.assertIs(self.cls(f'{os.devnull}\x00').is_char_device(), False) - def test_is_mount_root(self): + def test_is_mount(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_mount()) + self.assertFalse((P / 'dirA').is_mount()) + self.assertFalse((P / 'non-existing').is_mount()) + self.assertFalse((P / 'fileA' / 'bah').is_mount()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_mount()) if os.name == 'nt': R = self.cls('c:\\') else: @@ -1850,6 +1902,27 @@ def test_is_mount_root(self): self.assertTrue(R.is_mount()) self.assertFalse((R / '\udfff').is_mount()) + def test_samefile(self): + parser = self.parser + fileA_path = parser.join(self.base, 'fileA') + fileB_path = parser.join(self.base, 'dirB', 'fileB') + p = self.cls(fileA_path) + pp = self.cls(fileA_path) + q = self.cls(fileB_path) + self.assertTrue(p.samefile(fileA_path)) + self.assertTrue(p.samefile(pp)) + self.assertFalse(p.samefile(fileB_path)) + self.assertFalse(p.samefile(q)) + # Test the non-existent file case + non_existent = parser.join(self.base, 'foo') + r = self.cls(non_existent) + self.assertRaises(FileNotFoundError, p.samefile, r) + self.assertRaises(FileNotFoundError, p.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, p) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, r) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + def test_passing_kwargs_errors(self): with self.assertRaises(TypeError): self.cls(foo="bar") diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index dd9425ce393623..f4c364c6fe5109 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1300,15 +1300,9 @@ def test_unsupported_operation(self): e = UnsupportedOperation self.assertRaises(e, p.stat) self.assertRaises(e, p.exists) - self.assertRaises(e, p.samefile, 'foo') self.assertRaises(e, p.is_dir) self.assertRaises(e, p.is_file) - self.assertRaises(e, p.is_mount) self.assertRaises(e, p.is_symlink) - self.assertRaises(e, p.is_block_device) - self.assertRaises(e, p.is_char_device) - self.assertRaises(e, p.is_fifo) - self.assertRaises(e, p.is_socket) self.assertRaises(e, p.open) self.assertRaises(e, p.read_bytes) self.assertRaises(e, p.read_text) @@ -1535,27 +1529,6 @@ def assertEqualNormCase(self, path_a, path_b): normcase = self.parser.normcase self.assertEqual(normcase(path_a), normcase(path_b)) - def test_samefile(self): - parser = self.parser - fileA_path = parser.join(self.base, 'fileA') - fileB_path = parser.join(self.base, 'dirB', 'fileB') - p = self.cls(fileA_path) - pp = self.cls(fileA_path) - q = self.cls(fileB_path) - self.assertTrue(p.samefile(fileA_path)) - self.assertTrue(p.samefile(pp)) - self.assertFalse(p.samefile(fileB_path)) - self.assertFalse(p.samefile(q)) - # Test the non-existent file case - non_existent = parser.join(self.base, 'foo') - r = self.cls(non_existent) - self.assertRaises(FileNotFoundError, p.samefile, r) - self.assertRaises(FileNotFoundError, p.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, p) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, r) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - def test_exists(self): P = self.cls p = P(self.base) @@ -2115,15 +2088,6 @@ def test_is_file_no_follow_symlinks(self): self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) - def test_is_mount(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_mount()) - self.assertFalse((P / 'dirA').is_mount()) - self.assertFalse((P / 'non-existing').is_mount()) - self.assertFalse((P / 'fileA' / 'bah').is_mount()) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_mount()) - def test_is_symlink(self): P = self.cls(self.base) self.assertFalse((P / 'fileA').is_symlink()) @@ -2140,51 +2104,6 @@ def test_is_symlink(self): self.assertIs((P / 'linkA\udfff').is_file(), False) self.assertIs((P / 'linkA\x00').is_file(), False) - def test_is_junction_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_junction()) - self.assertFalse((P / 'dirA').is_junction()) - self.assertFalse((P / 'non-existing').is_junction()) - self.assertFalse((P / 'fileA' / 'bah').is_junction()) - self.assertFalse((P / 'fileA\udfff').is_junction()) - self.assertFalse((P / 'fileA\x00').is_junction()) - - def test_is_fifo_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_fifo()) - self.assertFalse((P / 'dirA').is_fifo()) - self.assertFalse((P / 'non-existing').is_fifo()) - self.assertFalse((P / 'fileA' / 'bah').is_fifo()) - self.assertIs((P / 'fileA\udfff').is_fifo(), False) - self.assertIs((P / 'fileA\x00').is_fifo(), False) - - def test_is_socket_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_socket()) - self.assertFalse((P / 'dirA').is_socket()) - self.assertFalse((P / 'non-existing').is_socket()) - self.assertFalse((P / 'fileA' / 'bah').is_socket()) - self.assertIs((P / 'fileA\udfff').is_socket(), False) - self.assertIs((P / 'fileA\x00').is_socket(), False) - - def test_is_block_device_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_block_device()) - self.assertFalse((P / 'dirA').is_block_device()) - self.assertFalse((P / 'non-existing').is_block_device()) - self.assertFalse((P / 'fileA' / 'bah').is_block_device()) - self.assertIs((P / 'fileA\udfff').is_block_device(), False) - self.assertIs((P / 'fileA\x00').is_block_device(), False) - - def test_is_char_device_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_char_device()) - self.assertFalse((P / 'dirA').is_char_device()) - self.assertFalse((P / 'non-existing').is_char_device()) - self.assertFalse((P / 'fileA' / 'bah').is_char_device()) - self.assertIs((P / 'fileA\udfff').is_char_device(), False) - self.assertIs((P / 'fileA\x00').is_char_device(), False) - def test_delete_file(self): p = self.cls(self.base) / 'fileA' p._delete() From db9bea0386c1c0b6c9d7c66474cda7e47e4b56f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Srinivas=20Reddy=20Thatiparthy=20=28=E0=B0=A4=E0=B0=BE?= =?UTF-8?q?=E0=B0=9F=E0=B0=BF=E0=B0=AA=E0=B0=B0=E0=B1=8D=E0=B0=A4=E0=B0=BF?= =?UTF-8?q?=20=E0=B0=B6=E0=B1=8D=E0=B0=B0=E0=B1=80=E0=B0=A8=E0=B0=BF?= =?UTF-8?q?=E0=B0=B5=E0=B0=BE=E0=B0=B8=E0=B1=8D=20=20=E0=B0=B0=E0=B1=86?= =?UTF-8?q?=E0=B0=A1=E0=B1=8D=E0=B0=A1=E0=B0=BF=29?= Date: Wed, 11 Dec 2024 13:05:17 +0530 Subject: [PATCH 165/427] gh-127740: For odd-length input to bytes.fromhex(...) change the error message to ValueError: fromhex() arg must be of even length (#127756) --- Lib/test/test_bytes.py | 6 ++++++ ...4-12-10-21-08-05.gh-issue-127740.0tWC9h.rst | 3 +++ Objects/bytesobject.c | 18 ++++++++++++++---- 3 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 9e1985bb3a7639..32cd178fa3b445 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -459,6 +459,12 @@ def test_fromhex(self): self.assertRaises(ValueError, self.type2test.fromhex, '\x00') self.assertRaises(ValueError, self.type2test.fromhex, '12 \x00 34') + # For odd number of character(s) + for value in ("a", "aaa", "deadbee"): + with self.assertRaises(ValueError) as cm: + self.type2test.fromhex(value) + self.assertIn("fromhex() arg must contain an even number of hexadecimal digits", str(cm.exception)) + for data, pos in ( # invalid first hexadecimal character ('12 x4 56', 3), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst new file mode 100644 index 00000000000000..f614dbb59bdc87 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst @@ -0,0 +1,3 @@ +Fix error message in :func:`bytes.fromhex` when given an odd number of +digits to properly indicate that an even number of hexadecimal digits is +required. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 8c7651f0f3aa45..533089d25cd73a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2543,7 +2543,12 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) bot = _PyLong_DigitValue[*str]; if (bot >= 16) { - invalid_char = str - PyUnicode_1BYTE_DATA(string); + /* Check if we had a second digit */ + if (str >= end){ + invalid_char = -1; + } else { + invalid_char = str - PyUnicode_1BYTE_DATA(string); + } goto error; } str++; @@ -2554,9 +2559,14 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) return _PyBytesWriter_Finish(&writer, buf); error: - PyErr_Format(PyExc_ValueError, - "non-hexadecimal number found in " - "fromhex() arg at position %zd", invalid_char); + if (invalid_char == -1) { + PyErr_SetString(PyExc_ValueError, + "fromhex() arg must contain an even number of hexadecimal digits"); + } else { + PyErr_Format(PyExc_ValueError, + "non-hexadecimal number found in " + "fromhex() arg at position %zd", invalid_char); + } _PyBytesWriter_Dealloc(&writer); return NULL; } From 2cdeb61b57e638ae46a04386330a12abe9cddf2c Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Wed, 11 Dec 2024 04:27:07 -0500 Subject: [PATCH 166/427] Add `extern "C"` around `PyTraceMalloc_` functions. (#127772) Pretty much everything else exported by Python.h has an extern "C" annotation, yet this header appears to be missing one. --- Include/cpython/tracemalloc.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Include/cpython/tracemalloc.h b/Include/cpython/tracemalloc.h index 61a16ea9a9f3eb..6d094291ae2e90 100644 --- a/Include/cpython/tracemalloc.h +++ b/Include/cpython/tracemalloc.h @@ -1,6 +1,9 @@ #ifndef Py_LIMITED_API #ifndef Py_TRACEMALLOC_H #define Py_TRACEMALLOC_H +#ifdef __cplusplus +extern "C" { +#endif /* Track an allocated memory block in the tracemalloc module. Return 0 on success, return -1 on error (failed to allocate memory to store @@ -22,5 +25,8 @@ PyAPI_FUNC(int) PyTraceMalloc_Untrack( unsigned int domain, uintptr_t ptr); +#ifdef __cplusplus +} +#endif #endif // !Py_TRACEMALLOC_H #endif // !Py_LIMITED_API From d5d84c3f13fe7fe591b375c41979d362bc11957a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 11 Dec 2024 06:14:04 -0500 Subject: [PATCH 167/427] gh-127791: Fix, document, and test `PyUnstable_AtExit` (#127793) --- Doc/c-api/init.rst | 9 ++++ Doc/c-api/sys.rst | 4 ++ Include/internal/pycore_atexit.h | 1 - ...-12-10-14-25-22.gh-issue-127791.YRw4GU.rst | 2 + Modules/_testcapimodule.c | 48 +++++++++++++++++++ Modules/_testinternalcapi.c | 34 ------------- Modules/atexitmodule.c | 12 +++-- 7 files changed, 71 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index ba1c2852f0bd53..dd63dd013e32dc 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -567,6 +567,15 @@ Initializing and finalizing the interpreter customized Python that always runs in isolated mode using :c:func:`Py_RunMain`. +.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) + + Register an :mod:`atexit` callback for the target interpreter *interp*. + This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and + data pointer for the callback. + + The :term:`GIL` must be held for *interp*. + + .. versionadded:: 3.13 Process-wide parameters ======================= diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index d6fca1a0b0a219..c688afdca8231d 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -426,3 +426,7 @@ Process Control function registered last is called first. Each cleanup function will be called at most once. Since Python's internal finalization will have completed before the cleanup function, no Python APIs should be called by *func*. + + .. seealso:: + + :c:func:`PyUnstable_AtExit` for passing a ``void *data`` argument. diff --git a/Include/internal/pycore_atexit.h b/Include/internal/pycore_atexit.h index 507a5c03cbc792..cde5b530baf00c 100644 --- a/Include/internal/pycore_atexit.h +++ b/Include/internal/pycore_atexit.h @@ -44,7 +44,6 @@ typedef struct { struct atexit_state { atexit_callback *ll_callbacks; - atexit_callback *last_ll_callback; // XXX The rest of the state could be moved to the atexit module state // and a low-level callback added for it during module exec. diff --git a/Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst b/Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst new file mode 100644 index 00000000000000..70751f18f5cf17 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst @@ -0,0 +1,2 @@ +Fix loss of callbacks after more than one call to +:c:func:`PyUnstable_AtExit`. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 26f68691e44f83..8d86b535effb9a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3353,6 +3353,53 @@ type_freeze(PyObject *module, PyObject *args) Py_RETURN_NONE; } +struct atexit_data { + int called; + PyThreadState *tstate; + PyInterpreterState *interp; +}; + +static void +atexit_callback(void *data) +{ + struct atexit_data *at_data = (struct atexit_data *)data; + // Ensure that the callback is from the same interpreter + assert(PyThreadState_Get() == at_data->tstate); + assert(PyInterpreterState_Get() == at_data->interp); + ++at_data->called; +} + +static PyObject * +test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *oldts = PyThreadState_Swap(NULL); + PyThreadState *tstate = Py_NewInterpreter(); + + struct atexit_data data = {0}; + data.tstate = PyThreadState_Get(); + data.interp = PyInterpreterState_Get(); + + int amount = 10; + for (int i = 0; i < amount; ++i) + { + int res = PyUnstable_AtExit(tstate->interp, atexit_callback, (void *)&data); + if (res < 0) { + Py_EndInterpreter(tstate); + PyThreadState_Swap(oldts); + PyErr_SetString(PyExc_RuntimeError, "atexit callback failed"); + return NULL; + } + } + + Py_EndInterpreter(tstate); + PyThreadState_Swap(oldts); + + if (data.called != amount) { + PyErr_SetString(PyExc_RuntimeError, "atexit callback not called"); + return NULL; + } + Py_RETURN_NONE; +} static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, @@ -3495,6 +3542,7 @@ static PyMethodDef TestMethods[] = { {"test_critical_sections", test_critical_sections, METH_NOARGS}, {"finalize_thread_hang", finalize_thread_hang, METH_O, NULL}, {"type_freeze", type_freeze, METH_VARARGS}, + {"test_atexit", test_atexit, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 1bb71a3e80b39d..288daf09a5fe5c 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1236,39 +1236,6 @@ unicode_transformdecimalandspacetoascii(PyObject *self, PyObject *arg) return _PyUnicode_TransformDecimalAndSpaceToASCII(arg); } - -struct atexit_data { - int called; -}; - -static void -callback(void *data) -{ - ((struct atexit_data *)data)->called += 1; -} - -static PyObject * -test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyThreadState *oldts = PyThreadState_Swap(NULL); - PyThreadState *tstate = Py_NewInterpreter(); - - struct atexit_data data = {0}; - int res = PyUnstable_AtExit(tstate->interp, callback, (void *)&data); - Py_EndInterpreter(tstate); - PyThreadState_Swap(oldts); - if (res < 0) { - return NULL; - } - - if (data.called == 0) { - PyErr_SetString(PyExc_RuntimeError, "atexit callback not called"); - return NULL; - } - Py_RETURN_NONE; -} - - static PyObject * test_pyobject_is_freed(const char *test_name, PyObject *op) { @@ -2128,7 +2095,6 @@ static PyMethodDef module_functions[] = { {"_PyTraceMalloc_GetTraceback", tracemalloc_get_traceback, METH_VARARGS}, {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL}, {"_PyUnicode_TransformDecimalAndSpaceToASCII", unicode_transformdecimalandspacetoascii, METH_O}, - {"test_atexit", test_atexit, METH_NOARGS}, {"check_pyobject_forbidden_bytes_is_freed", check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, {"check_pyobject_freed_is_freed", check_pyobject_freed_is_freed, METH_NOARGS}, diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 297a8d74ba3bf4..c009235b7a36c2 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -27,7 +27,10 @@ int PyUnstable_AtExit(PyInterpreterState *interp, atexit_datacallbackfunc func, void *data) { - assert(interp == _PyInterpreterState_GET()); + PyThreadState *tstate = _PyThreadState_GET(); + _Py_EnsureTstateNotNULL(tstate); + assert(tstate->interp == interp); + atexit_callback *callback = PyMem_Malloc(sizeof(atexit_callback)); if (callback == NULL) { PyErr_NoMemory(); @@ -38,12 +41,13 @@ PyUnstable_AtExit(PyInterpreterState *interp, callback->next = NULL; struct atexit_state *state = &interp->atexit; - if (state->ll_callbacks == NULL) { + atexit_callback *top = state->ll_callbacks; + if (top == NULL) { state->ll_callbacks = callback; - state->last_ll_callback = callback; } else { - state->last_ll_callback->next = callback; + callback->next = top; + state->ll_callbacks = callback; } return 0; } From ce76b547f94de6b1c9c74657b4e8f150365ad76f Mon Sep 17 00:00:00 2001 From: Justin Applegate <70449145+Legoclones@users.noreply.github.com> Date: Wed, 11 Dec 2024 05:37:59 -0700 Subject: [PATCH 168/427] gh-126992: Change pickle code to base 10 for load_long and load_int (GH-127042) Co-authored-by: Serhiy Storchaka --- Lib/pickle.py | 4 +- Lib/test/pickletester.py | 20 ++++++++++ Lib/test/test_pickletools.py | 37 +++++++++++++++++++ ...-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst | 1 + Modules/_pickle.c | 11 ++---- 5 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst diff --git a/Lib/pickle.py b/Lib/pickle.py index 25dadb3f75a573..1920973e3f83e9 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -1387,7 +1387,7 @@ def load_int(self): elif data == TRUE[1:]: val = True else: - val = int(data, 0) + val = int(data) self.append(val) dispatch[INT[0]] = load_int @@ -1407,7 +1407,7 @@ def load_long(self): val = self.readline()[:-1] if val and val[-1] == b'L'[0]: val = val[:-1] - self.append(int(val, 0)) + self.append(int(val)) dispatch[LONG[0]] = load_long def load_long1(self): diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index cf020a48b81cfa..bdc7ef62943a28 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1012,6 +1012,26 @@ def test_constants(self): self.assertIs(self.loads(b'I01\n.'), True) self.assertIs(self.loads(b'I00\n.'), False) + def test_zero_padded_integers(self): + self.assertEqual(self.loads(b'I010\n.'), 10) + self.assertEqual(self.loads(b'I-010\n.'), -10) + self.assertEqual(self.loads(b'I0010\n.'), 10) + self.assertEqual(self.loads(b'I-0010\n.'), -10) + self.assertEqual(self.loads(b'L010\n.'), 10) + self.assertEqual(self.loads(b'L-010\n.'), -10) + self.assertEqual(self.loads(b'L0010\n.'), 10) + self.assertEqual(self.loads(b'L-0010\n.'), -10) + self.assertEqual(self.loads(b'L010L\n.'), 10) + self.assertEqual(self.loads(b'L-010L\n.'), -10) + + def test_nondecimal_integers(self): + self.assertRaises(ValueError, self.loads, b'I0b10\n.') + self.assertRaises(ValueError, self.loads, b'I0o10\n.') + self.assertRaises(ValueError, self.loads, b'I0x10\n.') + self.assertRaises(ValueError, self.loads, b'L0b10L\n.') + self.assertRaises(ValueError, self.loads, b'L0o10L\n.') + self.assertRaises(ValueError, self.loads, b'L0x10L\n.') + def test_empty_bytestring(self): # issue 11286 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 265dc497ccb86c..a178d3353eecdf 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -443,6 +443,43 @@ def test_persid(self): highest protocol among opcodes = 0 ''') + def test_constants(self): + self.check_dis(b"(NI00\nI01\n\x89\x88t.", '''\ + 0: ( MARK + 1: N NONE + 2: I INT False + 6: I INT True + 10: \\x89 NEWFALSE + 11: \\x88 NEWTRUE + 12: t TUPLE (MARK at 0) + 13: . STOP +highest protocol among opcodes = 2 +''') + + def test_integers(self): + self.check_dis(b"(I0\nI1\nI10\nI011\nL12\nL13L\nL014\nL015L\nt.", '''\ + 0: ( MARK + 1: I INT 0 + 4: I INT 1 + 7: I INT 10 + 11: I INT 11 + 16: L LONG 12 + 20: L LONG 13 + 25: L LONG 14 + 30: L LONG 15 + 36: t TUPLE (MARK at 0) + 37: . STOP +highest protocol among opcodes = 0 +''') + + def test_nondecimal_integers(self): + self.check_dis_error(b'I0b10\n.', '', 'invalid literal for int') + self.check_dis_error(b'I0o10\n.', '', 'invalid literal for int') + self.check_dis_error(b'I0x10\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0b10L\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0o10L\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0x10L\n.', '', 'invalid literal for int') + class MiscTestCase(unittest.TestCase): def test__all__(self): diff --git a/Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst b/Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst new file mode 100644 index 00000000000000..526785f68cc807 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst @@ -0,0 +1 @@ +Fix LONG and INT opcodes to only use base 10 for string to integer conversion in :mod:`pickle`. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 2696f38046121f..599b5f92c2a1f7 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -5211,16 +5211,14 @@ load_int(PickleState *state, UnpicklerObject *self) return bad_readline(state); errno = 0; - /* XXX: Should the base argument of strtol() be explicitly set to 10? - XXX(avassalotti): Should this uses PyOS_strtol()? */ - x = strtol(s, &endptr, 0); + /* XXX(avassalotti): Should this uses PyOS_strtol()? */ + x = strtol(s, &endptr, 10); if (errno || (*endptr != '\n' && *endptr != '\0')) { /* Hm, maybe we've got something long. Let's try reading * it as a Python int object. */ errno = 0; - /* XXX: Same thing about the base here. */ - value = PyLong_FromString(s, NULL, 0); + value = PyLong_FromString(s, NULL, 10); if (value == NULL) { PyErr_SetString(PyExc_ValueError, "could not convert string to int"); @@ -5370,8 +5368,7 @@ load_long(PickleState *state, UnpicklerObject *self) the 'L' to be present. */ if (s[len-2] == 'L') s[len-2] = '\0'; - /* XXX: Should the base argument explicitly set to 10? */ - value = PyLong_FromString(s, NULL, 0); + value = PyLong_FromString(s, NULL, 10); if (value == NULL) return -1; From b2ad7e0a2c1518539d9b0ef83c9f7a09d10fd303 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:57:51 +0200 Subject: [PATCH 169/427] CI: Use bash to properly expand variable (#127822) --- .github/workflows/reusable-windows-msi.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index d0d53dba0b45d1..a1c45d954247fb 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -24,4 +24,5 @@ jobs: with: persist-credentials: false - name: Build CPython installer - run: .\Tools\msi\build.bat --doc -"${ARCH}" + run: ./Tools/msi/build.bat --doc -"${ARCH}" + shell: bash From 359389ed51aecc107681e600b71852c0a97304e1 Mon Sep 17 00:00:00 2001 From: Nano Date: Wed, 11 Dec 2024 21:28:19 +0800 Subject: [PATCH 170/427] gh-123401: Fix http.cookies module to support obsolete RFC 850 date format (#123405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Wulian <1055917385@qq.com> Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Victor Stinner --- Lib/http/cookies.py | 6 ++- Lib/test/test_http_cookies.py | 46 +++++++++++++++++++ ...-08-27-18-58-01.gh-issue-123401.t4-FpI.rst | 3 ++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index d7e8d08b2d92c1..23d5461f86fc23 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -425,9 +425,11 @@ def OutputString(self, attrs=None): ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P # Start of group 'val' - "(?:[^\\"]|\\.)*" # Any doublequoted string + "(?:[^\\"]|\\.)*" # Any double-quoted string | # or - \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr + # Special case for "expires" attr + (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day + [\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Date and time in specific format | # or [""" + _LegalValueChars + r"""]* # Any word or empty string ) # End of group 'val' diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 8879902a6e2f41..7b3dc0fdaedc3b 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -59,6 +59,52 @@ def test_basic(self): for k, v in sorted(case['dict'].items()): self.assertEqual(C[k].value, v) + def test_obsolete_rfc850_date_format(self): + # Test cases with different days and dates in obsolete RFC 850 format + test_cases = [ + # from RFC 850, change EST to GMT + # https://datatracker.ietf.org/doc/html/rfc850#section-2 + { + 'data': 'key=value; expires=Saturday, 01-Jan-83 00:00:00 GMT', + 'output': 'Saturday, 01-Jan-83 00:00:00 GMT' + }, + { + 'data': 'key=value; expires=Friday, 19-Nov-82 16:59:30 GMT', + 'output': 'Friday, 19-Nov-82 16:59:30 GMT' + }, + # from RFC 9110 + # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.6.7-6 + { + 'data': 'key=value; expires=Sunday, 06-Nov-94 08:49:37 GMT', + 'output': 'Sunday, 06-Nov-94 08:49:37 GMT' + }, + # other test cases + { + 'data': 'key=value; expires=Wednesday, 09-Nov-94 08:49:37 GMT', + 'output': 'Wednesday, 09-Nov-94 08:49:37 GMT' + }, + { + 'data': 'key=value; expires=Friday, 11-Nov-94 08:49:37 GMT', + 'output': 'Friday, 11-Nov-94 08:49:37 GMT' + }, + { + 'data': 'key=value; expires=Monday, 14-Nov-94 08:49:37 GMT', + 'output': 'Monday, 14-Nov-94 08:49:37 GMT' + }, + ] + + for case in test_cases: + with self.subTest(data=case['data']): + C = cookies.SimpleCookie() + C.load(case['data']) + + # Extract the cookie name from the data string + cookie_name = case['data'].split('=')[0] + + # Check if the cookie is loaded correctly + self.assertIn(cookie_name, C) + self.assertEqual(C[cookie_name].get('expires'), case['output']) + def test_unquote(self): cases = [ (r'a="b=\""', 'b="'), diff --git a/Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst b/Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst new file mode 100644 index 00000000000000..638f3f76239ca6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst @@ -0,0 +1,3 @@ +The :mod:`http.cookies` module now supports parsing obsolete :rfc:`850` +date formats, in accordance with :rfc:`9110` requirements. +Patch by Nano Zheng. From 5a23994a3dbee43a0b08f5920032f60f38b63071 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 11 Dec 2024 14:02:59 +0000 Subject: [PATCH 171/427] GH-127058: Make `PySequence_Tuple` safer and probably faster. (#127758) * Use a small buffer, then list when constructing a tuple from an arbitrary sequence. --- Include/internal/pycore_list.h | 2 +- Lib/test/test_capi/test_tuple.py | 25 ++++++ ...-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst | 3 + Objects/abstract.c | 87 +++++++++---------- Objects/listobject.c | 19 ++++ 5 files changed, 88 insertions(+), 48 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index f03e484f5ef8b0..836ff30abfcedb 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -62,7 +62,7 @@ typedef struct { union _PyStackRef; PyAPI_FUNC(PyObject *)_PyList_FromStackRefSteal(const union _PyStackRef *src, Py_ssize_t n); - +PyAPI_FUNC(PyObject *)_PyList_AsTupleAndClear(PyListObject *v); #ifdef __cplusplus } diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index e6b49caeb51f32..6349467c5d6b70 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -1,5 +1,6 @@ import unittest import sys +import gc from collections import namedtuple from test.support import import_helper @@ -257,5 +258,29 @@ def test__tuple_resize(self): self.assertRaises(SystemError, resize, [1, 2, 3], 0, False) self.assertRaises(SystemError, resize, NULL, 0, False) + def test_bug_59313(self): + # Before 3.14, the C-API function PySequence_Tuple + # would create incomplete tuples which were visible to + # the cycle GC, and this test would crash the interpeter. + TAG = object() + tuples = [] + + def referrer_tuples(): + return [x for x in gc.get_referrers(TAG) + if isinstance(x, tuple)] + + def my_iter(): + nonlocal tuples + yield TAG # 'tag' gets stored in the result tuple + tuples += referrer_tuples() + for x in range(10): + tuples += referrer_tuples() + # Prior to 3.13 would raise a SystemError when the tuple needs to be resized + yield x + + self.assertEqual(tuple(my_iter()), (TAG, *range(10))) + self.assertEqual(tuples, []) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst new file mode 100644 index 00000000000000..248e1b4855afb8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst @@ -0,0 +1,3 @@ +``PySequence_Tuple`` now creates the resulting tuple atomically, preventing +partially created tuples being visible to the garbage collector or through +``gc.get_referrers()`` diff --git a/Objects/abstract.c b/Objects/abstract.c index f6647874d732f6..c92ef10aa79648 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1993,9 +1993,6 @@ PyObject * PySequence_Tuple(PyObject *v) { PyObject *it; /* iter(v) */ - Py_ssize_t n; /* guess for result tuple size */ - PyObject *result = NULL; - Py_ssize_t j; if (v == NULL) { return null_error(); @@ -2017,58 +2014,54 @@ PySequence_Tuple(PyObject *v) if (it == NULL) return NULL; - /* Guess result size and allocate space. */ - n = PyObject_LengthHint(v, 10); - if (n == -1) - goto Fail; - result = PyTuple_New(n); - if (result == NULL) - goto Fail; - - /* Fill the tuple. */ - for (j = 0; ; ++j) { + Py_ssize_t n; + PyObject *buffer[8]; + for (n = 0; n < 8; n++) { PyObject *item = PyIter_Next(it); if (item == NULL) { - if (PyErr_Occurred()) - goto Fail; - break; - } - if (j >= n) { - size_t newn = (size_t)n; - /* The over-allocation strategy can grow a bit faster - than for lists because unlike lists the - over-allocation isn't permanent -- we reclaim - the excess before the end of this routine. - So, grow by ten and then add 25%. - */ - newn += 10u; - newn += newn >> 2; - if (newn > PY_SSIZE_T_MAX) { - /* Check for overflow */ - PyErr_NoMemory(); - Py_DECREF(item); - goto Fail; + if (PyErr_Occurred()) { + goto fail; } - n = (Py_ssize_t)newn; - if (_PyTuple_Resize(&result, n) != 0) { - Py_DECREF(item); - goto Fail; + Py_DECREF(it); + return _PyTuple_FromArraySteal(buffer, n); + } + buffer[n] = item; + } + PyListObject *list = (PyListObject *)PyList_New(16); + if (list == NULL) { + goto fail; + } + assert(n == 8); + Py_SET_SIZE(list, n); + for (Py_ssize_t j = 0; j < n; j++) { + PyList_SET_ITEM(list, j, buffer[j]); + } + for (;;) { + PyObject *item = PyIter_Next(it); + if (item == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(list); + Py_DECREF(it); + return NULL; } + break; + } + if (_PyList_AppendTakeRef(list, item) < 0) { + Py_DECREF(list); + Py_DECREF(it); + return NULL; } - PyTuple_SET_ITEM(result, j, item); } - - /* Cut tuple back if guess was too large. */ - if (j < n && - _PyTuple_Resize(&result, j) != 0) - goto Fail; - Py_DECREF(it); - return result; - -Fail: - Py_XDECREF(result); + PyObject *res = _PyList_AsTupleAndClear(list); + Py_DECREF(list); + return res; +fail: Py_DECREF(it); + while (n > 0) { + n--; + Py_DECREF(buffer[n]); + } return NULL; } diff --git a/Objects/listobject.c b/Objects/listobject.c index 3832295600a0ab..a877bad66be45f 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3174,6 +3174,25 @@ PyList_AsTuple(PyObject *v) return ret; } +PyObject * +_PyList_AsTupleAndClear(PyListObject *self) +{ + assert(self != NULL); + PyObject *ret; + if (self->ob_item == NULL) { + return PyTuple_New(0); + } + Py_BEGIN_CRITICAL_SECTION(self); + PyObject **items = self->ob_item; + Py_ssize_t size = Py_SIZE(self); + self->ob_item = NULL; + Py_SET_SIZE(self, 0); + ret = _PyTuple_FromArraySteal(items, size); + free_list_items(items, false); + Py_END_CRITICAL_SECTION(); + return ret; +} + PyObject * _PyList_FromStackRefSteal(const _PyStackRef *src, Py_ssize_t n) { From b0f278ff0551b06191cec01445c577e3b25570da Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Wed, 11 Dec 2024 16:06:07 +0100 Subject: [PATCH 172/427] gh-127065: Make methodcaller thread-safe and re-entrant (GH-127746) The function `operator.methodcaller` was not thread-safe since the additional of the vectorcall method in gh-89013. In the free threading build the issue is easy to trigger, for the normal build harder. This makes the `methodcaller` safe by: * Replacing the lazy initialization with initialization in the constructor. * Using a stack allocated space for the vectorcall arguments and falling back to `tp_call` for calls with more than 8 arguments. --- .../test_free_threading/test_methodcaller.py | 33 ++++ Lib/test/test_operator.py | 13 ++ ...-12-01-22-28-41.gh-issue-127065.tFpRer.rst | 1 + Modules/_operator.c | 180 ++++++++---------- 4 files changed, 131 insertions(+), 96 deletions(-) create mode 100644 Lib/test/test_free_threading/test_methodcaller.py create mode 100644 Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst diff --git a/Lib/test/test_free_threading/test_methodcaller.py b/Lib/test/test_free_threading/test_methodcaller.py new file mode 100644 index 00000000000000..8846b0010012f2 --- /dev/null +++ b/Lib/test/test_free_threading/test_methodcaller.py @@ -0,0 +1,33 @@ +import unittest +from threading import Thread +from test.support import threading_helper +from operator import methodcaller + + +class TestMethodcaller(unittest.TestCase): + def test_methodcaller_threading(self): + number_of_threads = 10 + size = 4_000 + + mc = methodcaller("append", 2) + + def work(mc, l, ii): + for _ in range(ii): + mc(l) + + worker_threads = [] + lists = [] + for ii in range(number_of_threads): + l = [] + lists.append(l) + worker_threads.append(Thread(target=work, args=[mc, l, size])) + for t in worker_threads: + t.start() + for t in worker_threads: + t.join() + for l in lists: + assert len(l) == size + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 812d46482e238a..82578a0ef1e6f2 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -482,6 +482,8 @@ def bar(self, f=42): return f def baz(*args, **kwds): return kwds['name'], kwds['self'] + def return_arguments(self, *args, **kwds): + return args, kwds a = A() f = operator.methodcaller('foo') self.assertRaises(IndexError, f, a) @@ -498,6 +500,17 @@ def baz(*args, **kwds): f = operator.methodcaller('baz', name='spam', self='eggs') self.assertEqual(f(a), ('spam', 'eggs')) + many_positional_arguments = tuple(range(10)) + many_kw_arguments = dict(zip('abcdefghij', range(10))) + f = operator.methodcaller('return_arguments', *many_positional_arguments) + self.assertEqual(f(a), (many_positional_arguments, {})) + + f = operator.methodcaller('return_arguments', **many_kw_arguments) + self.assertEqual(f(a), ((), many_kw_arguments)) + + f = operator.methodcaller('return_arguments', *many_positional_arguments, **many_kw_arguments) + self.assertEqual(f(a), (many_positional_arguments, many_kw_arguments)) + def test_inplace(self): operator = self.module class C(object): diff --git a/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst b/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst new file mode 100644 index 00000000000000..03d6953b9ddfa1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst @@ -0,0 +1 @@ +Make :func:`operator.methodcaller` thread-safe and re-entrant safe. diff --git a/Modules/_operator.c b/Modules/_operator.c index 6c1945174ab7cd..ce3ef015710223 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1595,78 +1595,75 @@ static PyType_Spec attrgetter_type_spec = { typedef struct { PyObject_HEAD PyObject *name; - PyObject *xargs; // reference to arguments passed in constructor + PyObject *args; PyObject *kwds; - PyObject **vectorcall_args; /* Borrowed references */ + PyObject *vectorcall_args; PyObject *vectorcall_kwnames; vectorcallfunc vectorcall; } methodcallerobject; -#ifndef Py_GIL_DISABLED -static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) -{ - PyObject* args = mc->xargs; - PyObject* kwds = mc->kwds; - - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - assert(nargs > 0); - mc->vectorcall_args = PyMem_Calloc( - nargs + (kwds ? PyDict_Size(kwds) : 0), - sizeof(PyObject*)); - if (!mc->vectorcall_args) { - PyErr_NoMemory(); - return -1; - } - /* The first item of vectorcall_args will be filled with obj later */ - if (nargs > 1) { - memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), - nargs * sizeof(PyObject*)); - } - if (kwds) { - const Py_ssize_t nkwds = PyDict_Size(kwds); - - mc->vectorcall_kwnames = PyTuple_New(nkwds); - if (!mc->vectorcall_kwnames) { - return -1; - } - Py_ssize_t i = 0, ppos = 0; - PyObject* key, * value; - while (PyDict_Next(kwds, &ppos, &key, &value)) { - PyTuple_SET_ITEM(mc->vectorcall_kwnames, i, Py_NewRef(key)); - mc->vectorcall_args[nargs + i] = value; // borrowed reference - ++i; - } - } - else { - mc->vectorcall_kwnames = NULL; - } - return 1; -} +#define _METHODCALLER_MAX_ARGS 8 static PyObject * -methodcaller_vectorcall( - methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) +methodcaller_vectorcall(methodcallerobject *mc, PyObject *const *args, + size_t nargsf, PyObject* kwnames) { if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) || !_PyArg_NoKwnames("methodcaller", kwnames)) { return NULL; } - if (mc->vectorcall_args == NULL) { - if (_methodcaller_initialize_vectorcall(mc) < 0) { - return NULL; - } - } + assert(mc->vectorcall_args != NULL); + + PyObject *tmp_args[_METHODCALLER_MAX_ARGS]; + tmp_args[0] = args[0]; + assert(1 + PyTuple_GET_SIZE(mc->vectorcall_args) <= _METHODCALLER_MAX_ARGS); + memcpy(tmp_args + 1, _PyTuple_ITEMS(mc->vectorcall_args), sizeof(PyObject *) * PyTuple_GET_SIZE(mc->vectorcall_args)); - assert(mc->vectorcall_args != 0); - mc->vectorcall_args[0] = args[0]; - return PyObject_VectorcallMethod( - mc->name, mc->vectorcall_args, - (PyTuple_GET_SIZE(mc->xargs)) | PY_VECTORCALL_ARGUMENTS_OFFSET, + return PyObject_VectorcallMethod(mc->name, tmp_args, + (1 + PyTuple_GET_SIZE(mc->args)) | PY_VECTORCALL_ARGUMENTS_OFFSET, mc->vectorcall_kwnames); } -#endif +static int +_methodcaller_initialize_vectorcall(methodcallerobject* mc) +{ + PyObject* args = mc->args; + PyObject* kwds = mc->kwds; + + if (kwds && PyDict_Size(kwds)) { + PyObject *values = PyDict_Values(kwds); + if (!values) { + return -1; + } + PyObject *values_tuple = PySequence_Tuple(values); + Py_DECREF(values); + if (!values_tuple) { + return -1; + } + if (PyTuple_GET_SIZE(args)) { + mc->vectorcall_args = PySequence_Concat(args, values_tuple); + Py_DECREF(values_tuple); + if (mc->vectorcall_args == NULL) { + return -1; + } + } + else { + mc->vectorcall_args = values_tuple; + } + mc->vectorcall_kwnames = PySequence_Tuple(kwds); + if (!mc->vectorcall_kwnames) { + return -1; + } + } + else { + mc->vectorcall_args = Py_NewRef(args); + mc->vectorcall_kwnames = NULL; + } + + mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; + return 0; +} /* AC 3.5: variable number of arguments, not currently support by AC */ static PyObject * @@ -1694,25 +1691,30 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (mc == NULL) { return NULL; } + mc->vectorcall = NULL; + mc->vectorcall_args = NULL; + mc->vectorcall_kwnames = NULL; + mc->kwds = Py_XNewRef(kwds); Py_INCREF(name); PyInterpreterState *interp = _PyInterpreterState_GET(); _PyUnicode_InternMortal(interp, &name); mc->name = name; - mc->xargs = Py_XNewRef(args); // allows us to use borrowed references - mc->kwds = Py_XNewRef(kwds); - mc->vectorcall_args = 0; - + mc->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); + if (mc->args == NULL) { + Py_DECREF(mc); + return NULL; + } -#ifdef Py_GIL_DISABLED - // gh-127065: The current implementation of methodcaller_vectorcall - // is not thread-safe because it modifies the `vectorcall_args` array, - // which is shared across calls. - mc->vectorcall = NULL; -#else - mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; -#endif + Py_ssize_t vectorcall_size = PyTuple_GET_SIZE(args) + + (kwds ? PyDict_Size(kwds) : 0); + if (vectorcall_size < (_METHODCALLER_MAX_ARGS)) { + if (_methodcaller_initialize_vectorcall(mc) < 0) { + Py_DECREF(mc); + return NULL; + } + } PyObject_GC_Track(mc); return (PyObject *)mc; @@ -1722,13 +1724,10 @@ static void methodcaller_clear(methodcallerobject *mc) { Py_CLEAR(mc->name); - Py_CLEAR(mc->xargs); + Py_CLEAR(mc->args); Py_CLEAR(mc->kwds); - if (mc->vectorcall_args != NULL) { - PyMem_Free(mc->vectorcall_args); - mc->vectorcall_args = 0; - Py_CLEAR(mc->vectorcall_kwnames); - } + Py_CLEAR(mc->vectorcall_args); + Py_CLEAR(mc->vectorcall_kwnames); } static void @@ -1745,8 +1744,10 @@ static int methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) { Py_VISIT(mc->name); - Py_VISIT(mc->xargs); + Py_VISIT(mc->args); Py_VISIT(mc->kwds); + Py_VISIT(mc->vectorcall_args); + Py_VISIT(mc->vectorcall_kwnames); Py_VISIT(Py_TYPE(mc)); return 0; } @@ -1765,15 +1766,7 @@ methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) if (method == NULL) return NULL; - - PyObject *cargs = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); - if (cargs == NULL) { - Py_DECREF(method); - return NULL; - } - - result = PyObject_Call(method, cargs, mc->kwds); - Py_DECREF(cargs); + result = PyObject_Call(method, mc->args, mc->kwds); Py_DECREF(method); return result; } @@ -1791,7 +1784,7 @@ methodcaller_repr(methodcallerobject *mc) } numkwdargs = mc->kwds != NULL ? PyDict_GET_SIZE(mc->kwds) : 0; - numposargs = PyTuple_GET_SIZE(mc->xargs) - 1; + numposargs = PyTuple_GET_SIZE(mc->args); numtotalargs = numposargs + numkwdargs; if (numtotalargs == 0) { @@ -1807,7 +1800,7 @@ methodcaller_repr(methodcallerobject *mc) } for (i = 0; i < numposargs; ++i) { - PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->xargs, i+1)); + PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->args, i)); if (onerepr == NULL) goto done; PyTuple_SET_ITEM(argreprs, i, onerepr); @@ -1859,14 +1852,14 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) { if (!mc->kwds || PyDict_GET_SIZE(mc->kwds) == 0) { Py_ssize_t i; - Py_ssize_t newarg_size = PyTuple_GET_SIZE(mc->xargs); - PyObject *newargs = PyTuple_New(newarg_size); + Py_ssize_t callargcount = PyTuple_GET_SIZE(mc->args); + PyObject *newargs = PyTuple_New(1 + callargcount); if (newargs == NULL) return NULL; PyTuple_SET_ITEM(newargs, 0, Py_NewRef(mc->name)); - for (i = 1; i < newarg_size; ++i) { - PyObject *arg = PyTuple_GET_ITEM(mc->xargs, i); - PyTuple_SET_ITEM(newargs, i, Py_NewRef(arg)); + for (i = 0; i < callargcount; ++i) { + PyObject *arg = PyTuple_GET_ITEM(mc->args, i); + PyTuple_SET_ITEM(newargs, i + 1, Py_NewRef(arg)); } return Py_BuildValue("ON", Py_TYPE(mc), newargs); } @@ -1884,12 +1877,7 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) constructor = PyObject_VectorcallDict(partial, newargs, 2, mc->kwds); Py_DECREF(partial); - PyObject *args = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); - if (!args) { - Py_DECREF(constructor); - return NULL; - } - return Py_BuildValue("NO", constructor, args); + return Py_BuildValue("NO", constructor, mc->args); } } From dd9da738ad1d420fabafaded3fe63912b2b17cfb Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 11 Dec 2024 11:28:44 -0500 Subject: [PATCH 173/427] gh-118915: C API: Document frame locals proxies. (#127720) Co-authored-by: Alex Waygood --- Doc/c-api/frame.rst | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 638a740e0c24da..1a52e146a69751 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -132,7 +132,7 @@ See also :ref:`Reflection `. .. versionadded:: 3.11 .. versionchanged:: 3.13 - As part of :pep:`667`, return a proxy object for optimized scopes. + As part of :pep:`667`, return an instance of :c:var:`PyFrameLocalsProxy_Type`. .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) @@ -140,6 +140,26 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. +Frame Locals Proxies +^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.13 + +The :attr:`~frame.f_locals` attribute on a :ref:`frame object ` +is an instance of a "frame-locals proxy". The proxy object exposes a +write-through view of the underlying locals dictionary for the frame. This +ensures that the variables exposed by ``f_locals`` are always up to date with +the live local variables in the frame itself. + +See :pep:`667` for more information. + +.. c:var:: PyTypeObject PyFrameLocalsProxy_Type + + The type of frame :func:`locals` proxy objects. + +.. c:function:: int PyFrameLocalsProxy_Check(PyObject *obj) + + Return non-zero if *obj* is a frame :func:`locals` proxy. Internal Frames ^^^^^^^^^^^^^^^ From bc262de06b10a2d119c28bac75060bf00301697a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 11 Dec 2024 17:37:38 +0000 Subject: [PATCH 174/427] GH-125174: Mark objects as statically allocated. (#127797) * Set a bit in the unused part of the refcount on 64 bit machines and the free-threaded build. * Use the top of the refcount range on 32 bit machines --- Include/internal/pycore_object.h | 16 ++++++++++++- Include/object.h | 20 ++++++++++++---- Include/refcount.h | 36 +++++++++++++++++++++++++---- Lib/test/test_builtin.py | 2 +- Lib/test/test_capi/test_immortal.py | 16 +++++++++++++ Modules/_testinternalcapi.c | 10 ++++++++ Objects/object.c | 12 +++++++++- 7 files changed, 99 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 6b0b464a6fdb96..22de3c9d4e32ea 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -73,14 +73,24 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); #define _PyObject_HEAD_INIT(type) \ { \ .ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL, \ + .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \ .ob_type = (type) \ } #else +#if SIZEOF_VOID_P > 4 #define _PyObject_HEAD_INIT(type) \ { \ - .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \ + .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \ + .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \ .ob_type = (type) \ } +#else +#define _PyObject_HEAD_INIT(type) \ + { \ + .ob_refcnt = _Py_STATIC_IMMORTAL_INITIAL_REFCNT, \ + .ob_type = (type) \ + } +#endif #endif #define _PyVarObject_HEAD_INIT(type, size) \ { \ @@ -127,7 +137,11 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) _Py_AddRefTotal(_PyThreadState_GET(), n); #endif #if !defined(Py_GIL_DISABLED) +#if SIZEOF_VOID_P > 4 + op->ob_refcnt += (PY_UINT32_T)n; +#else op->ob_refcnt += n; +#endif #else if (_Py_IsOwnedByCurrentThread(op)) { uint32_t local = op->ob_ref_local; diff --git a/Include/object.h b/Include/object.h index 3876d8449afbe2..da7b3668c033f4 100644 --- a/Include/object.h +++ b/Include/object.h @@ -71,7 +71,7 @@ whose size is determined when the object is allocated. #define PyObject_HEAD_INIT(type) \ { \ 0, \ - 0, \ + _Py_STATICALLY_ALLOCATED_FLAG, \ { 0 }, \ 0, \ _Py_IMMORTAL_REFCNT_LOCAL, \ @@ -81,7 +81,7 @@ whose size is determined when the object is allocated. #else #define PyObject_HEAD_INIT(type) \ { \ - { _Py_IMMORTAL_INITIAL_REFCNT }, \ + { _Py_STATIC_IMMORTAL_INITIAL_REFCNT }, \ (type) \ }, #endif @@ -120,9 +120,19 @@ struct _object { __pragma(warning(disable: 4201)) #endif union { - Py_ssize_t ob_refcnt; #if SIZEOF_VOID_P > 4 - PY_UINT32_T ob_refcnt_split[2]; + PY_INT64_T ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ + struct { +# if PY_BIG_ENDIAN + PY_UINT32_T ob_flags; + PY_UINT32_T ob_refcnt; +# else + PY_UINT32_T ob_refcnt; + PY_UINT32_T ob_flags; +# endif + }; +#else + Py_ssize_t ob_refcnt; #endif }; #ifdef _MSC_VER @@ -142,7 +152,7 @@ struct _object { // trashcan mechanism as a linked list pointer and by the GC to store the // computed "gc_refs" refcount. uintptr_t ob_tid; - uint16_t _padding; + uint16_t ob_flags; PyMutex ob_mutex; // per-object lock uint8_t ob_gc_bits; // gc-related state uint32_t ob_ref_local; // local reference count diff --git a/Include/refcount.h b/Include/refcount.h index 141cbd34dd72e6..6908c426141378 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -19,6 +19,9 @@ immortal. The latter should be the only instances that require cleanup during runtime finalization. */ +/* Leave the low bits for refcount overflow for old stable ABI code */ +#define _Py_STATICALLY_ALLOCATED_FLAG (1 << 7) + #if SIZEOF_VOID_P > 4 /* In 64+ bit systems, any object whose 32 bit reference count is >= 2**31 @@ -39,7 +42,8 @@ beyond the refcount limit. Immortality checks for reference count decreases will be done by checking the bit sign flag in the lower 32 bits. */ -#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(3UL << 30)) +#define _Py_IMMORTAL_INITIAL_REFCNT (3UL << 30) +#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(_Py_IMMORTAL_INITIAL_REFCNT | (((Py_ssize_t)_Py_STATICALLY_ALLOCATED_FLAG) << 32))) #else /* @@ -54,8 +58,10 @@ immortality, but the execution would still be correct. Reference count increases and decreases will first go through an immortality check by comparing the reference count field to the minimum immortality refcount. */ -#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(3L << 29)) +#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(5L << 28)) #define _Py_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(1L << 30)) +#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(7L << 28)) +#define _Py_STATIC_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(6L << 28)) #endif // Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is @@ -123,10 +129,21 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) #define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) +static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op) +{ +#if defined(Py_GIL_DISABLED) || SIZEOF_VOID_P > 4 + return (op->ob_flags & _Py_STATICALLY_ALLOCATED_FLAG) != 0; +#else + return op->ob_refcnt >= _Py_STATIC_IMMORTAL_MINIMUM_REFCNT; +#endif +} +#define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op)) + // Py_SET_REFCNT() implementation for stable ABI PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { + assert(refcnt >= 0); #if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 // Stable ABI implements Py_SET_REFCNT() as a function call // on limited C API version 3.13 and newer. @@ -139,9 +156,12 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { if (_Py_IsImmortal(ob)) { return; } - #ifndef Py_GIL_DISABLED +#if SIZEOF_VOID_P > 4 + ob->ob_refcnt = (PY_UINT32_T)refcnt; +#else ob->ob_refcnt = refcnt; +#endif #else if (_Py_IsOwnedByCurrentThread(ob)) { if ((size_t)refcnt > (size_t)UINT32_MAX) { @@ -252,13 +272,13 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); } #elif SIZEOF_VOID_P > 4 - PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; + PY_UINT32_T cur_refcnt = op->ob_refcnt; if (((int32_t)cur_refcnt) < 0) { // the object is immortal _Py_INCREF_IMMORTAL_STAT_INC(); return; } - op->ob_refcnt_split[PY_BIG_ENDIAN] = cur_refcnt + 1; + op->ob_refcnt = cur_refcnt + 1; #else if (_Py_IsImmortal(op)) { _Py_INCREF_IMMORTAL_STAT_INC(); @@ -354,7 +374,13 @@ static inline void Py_DECREF(PyObject *op) #elif defined(Py_REF_DEBUG) static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) { +#if SIZEOF_VOID_P > 4 + /* If an object has been freed, it will have a negative full refcnt + * If it has not it been freed, will have a very large refcnt */ + if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (UINT32_MAX - (1<<20))) { +#else if (op->ob_refcnt <= 0) { +#endif _Py_NegativeRefcount(filename, lineno, op); } if (_Py_IsImmortal(op)) { diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index e51711d9b4f1a4..06df217881a52f 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2691,7 +2691,7 @@ def __del__(self): class ImmortalTests(unittest.TestCase): if sys.maxsize < (1 << 32): - IMMORTAL_REFCOUNT = 3 << 29 + IMMORTAL_REFCOUNT = 7 << 28 else: IMMORTAL_REFCOUNT = 3 << 30 diff --git a/Lib/test/test_capi/test_immortal.py b/Lib/test/test_capi/test_immortal.py index ef5d32b7f01935..3e36913ac301c3 100644 --- a/Lib/test/test_capi/test_immortal.py +++ b/Lib/test/test_capi/test_immortal.py @@ -2,6 +2,7 @@ from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') class TestCAPI(unittest.TestCase): @@ -11,6 +12,21 @@ def test_immortal_builtins(self): def test_immortal_small_ints(self): _testcapi.test_immortal_small_ints() +class TestInternalCAPI(unittest.TestCase): + + def test_immortal_builtins(self): + for obj in range(-5, 256): + self.assertTrue(_testinternalcapi.is_static_immortal(obj)) + self.assertTrue(_testinternalcapi.is_static_immortal(None)) + self.assertTrue(_testinternalcapi.is_static_immortal(False)) + self.assertTrue(_testinternalcapi.is_static_immortal(True)) + self.assertTrue(_testinternalcapi.is_static_immortal(...)) + self.assertTrue(_testinternalcapi.is_static_immortal(())) + for obj in range(300, 400): + self.assertFalse(_testinternalcapi.is_static_immortal(obj)) + for obj in ([], {}, set()): + self.assertFalse(_testinternalcapi.is_static_immortal(obj)) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 288daf09a5fe5c..014f89997f7f60 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2049,6 +2049,15 @@ get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size); } +static PyObject * +is_static_immortal(PyObject *self, PyObject *op) +{ + if (_Py_IsStaticImmortal(op)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2146,6 +2155,7 @@ static PyMethodDef module_functions[] = { {"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS}, {"has_deferred_refcount", has_deferred_refcount, METH_O}, {"get_tracked_heap_size", get_tracked_heap_size, METH_NOARGS}, + {"is_static_immortal", is_static_immortal, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/object.c b/Objects/object.c index 74f47fa4239032..c64675b5e1d6c2 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2475,10 +2475,16 @@ new_reference(PyObject *op) { // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 #if !defined(Py_GIL_DISABLED) +#if SIZEOF_VOID_P > 4 + op->ob_refcnt_full = 1; + assert(op->ob_refcnt == 1); + assert(op->ob_flags == 0); +#else op->ob_refcnt = 1; +#endif #else op->ob_tid = _Py_ThreadId(); - op->_padding = 0; + op->ob_flags = 0; op->ob_mutex = (PyMutex){ 0 }; op->ob_gc_bits = 0; op->ob_ref_local = 1; @@ -2515,6 +2521,10 @@ _Py_SetImmortalUntracked(PyObject *op) || PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL_STATIC); } #endif + // Check if already immortal to avoid degrading from static immortal to plain immortal + if (_Py_IsImmortal(op)) { + return; + } #ifdef Py_GIL_DISABLED op->ob_tid = _Py_UNOWNED_TID; op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; From e8f4e272cc828f2b79fa17fc6b9786bdddab7ce4 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Wed, 11 Dec 2024 19:32:54 +0100 Subject: [PATCH 175/427] gh-111609: Test `end_offset` in SyntaxError subclass (#127830) Test `end_offset` in SyntaxError subclass --- Lib/test/test_exceptions.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 5beeac3adfc065..6ccfa9575f8569 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -2274,6 +2274,21 @@ def test_range_of_offsets(self): self.assertIn(expected, err.getvalue()) the_exception = exc + def test_subclass(self): + class MySyntaxError(SyntaxError): + pass + + try: + raise MySyntaxError("bad bad", ("bad.py", 1, 2, "abcdefg", 1, 7)) + except SyntaxError as exc: + with support.captured_stderr() as err: + sys.__excepthook__(*sys.exc_info()) + self.assertIn(""" + File "bad.py", line 1 + abcdefg + ^^^^^ +""", err.getvalue()) + def test_encodings(self): self.addCleanup(unlink, TESTFN) source = ( From c84928ed6de105696be24859e03f3ab27e11daf6 Mon Sep 17 00:00:00 2001 From: mpage Date: Wed, 11 Dec 2024 15:18:22 -0800 Subject: [PATCH 176/427] gh-115999: Specialize `CALL_KW` in free-threaded builds (#127713) * Enable specialization of CALL_KW * Fix bug pushing frame in _PY_FRAME_KW `_PY_FRAME_KW` pushes a pointer to the new frame onto the stack for consumption by the next uop. When pushing the frame fails, we do not want to push the result, `NULL`, to the stack because it is not a valid stackref. This works in the default build because `PyStackRef_NULL` and `NULL` are the same value, so the `PyStackRef_XCLOSE()` in the error handler ignores it. In the free-threaded build the values are not the same; `PyStackRef_XCLOSE()` will attempt to decref a null pointer. --- Python/bytecodes.c | 9 +++++---- Python/executor_cases.c.h | 11 +++++++---- Python/generated_cases.c.h | 24 ++++++++++-------------- Python/specialize.c | 17 ++++------------- 4 files changed, 26 insertions(+), 35 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3d280941b35244..d0e4c2bc45489b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4311,7 +4311,7 @@ dummy_func( assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); @@ -4319,9 +4319,10 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. SYNC_SP(); - if (new_frame == NULL) { + if (temp == NULL) { ERROR_NO_POP(); } + new_frame = temp; } op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable[1], self_or_null[1], unused[oparg], kwnames -- callable[1], self_or_null[1], unused[oparg], kwnames)) { @@ -4372,7 +4373,7 @@ dummy_func( _PUSH_FRAME; specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable[1], self_or_null[1], args[oparg], kwnames -- callable[1], self_or_null[1], args[oparg], kwnames)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); @@ -4380,7 +4381,7 @@ dummy_func( } OPCODE_DEFERRED_INC(CALL_KW); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } macro(CALL_KW) = diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 987ff2e6419669..18f19773d25c90 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5235,7 +5235,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); @@ -5243,12 +5243,15 @@ PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; - stack_pointer += -2 - oparg; + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { JUMP_TO_ERROR(); } + new_frame = temp; + stack_pointer[0].bits = (uintptr_t)new_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 33f32aba1e5145..fc0f55555f5c36 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1895,7 +1895,7 @@ callable = &stack_pointer[-3 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1905,7 +1905,7 @@ } OPCODE_DEFERRED_INC(CALL_KW); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD_KW @@ -2093,7 +2093,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); @@ -2101,12 +2101,12 @@ PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; - stack_pointer += -2 - oparg; + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { goto error; } + new_frame = temp; } // _SAVE_RETURN_OFFSET { @@ -2123,8 +2123,6 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -2271,7 +2269,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); @@ -2279,12 +2277,12 @@ PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; - stack_pointer += -2 - oparg; + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { goto error; } + new_frame = temp; } // _SAVE_RETURN_OFFSET { @@ -2301,8 +2299,6 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); diff --git a/Python/specialize.c b/Python/specialize.c index d3fea717243847..fd182e7d7a9215 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2107,7 +2107,7 @@ specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } write_u32(cache->func_version, version); - instr->op.code = bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY; + specialize(instr, bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY); return 0; } @@ -2202,10 +2202,9 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[CALL_KW] == INLINE_CACHE_ENTRIES_CALL_KW); assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW); - _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; if (PyFunction_Check(callable)) { fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); @@ -2221,19 +2220,11 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) } } else { - instr->op.code = CALL_KW_NON_PY; + specialize(instr, CALL_KW_NON_PY); fail = 0; } if (fail) { - STAT_INC(CALL, failure); - assert(!PyErr_Occurred()); - instr->op.code = CALL_KW; - cache->counter = adaptive_counter_backoff(cache->counter); - } - else { - STAT_INC(CALL, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr); } } From 41f29e5d16c314790559e563ce5ca0334fcd54df Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 12 Dec 2024 02:11:00 +0100 Subject: [PATCH 177/427] gh-127146: Some expected failures in Emscripten time tests (#127843) Disables two tests in the test_time suite, and adjusts test_os to reflect precision limits in Emscripten. --- Lib/test/test_os.py | 28 +++++++++++++++++++++------- Lib/test/test_time.py | 4 ++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f3d2ceb263f6f4..8aac92934f6ac0 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -805,14 +805,28 @@ def _test_utime(self, set_time, filename=None): set_time(filename, (atime_ns, mtime_ns)) st = os.stat(filename) - if support_subsecond: - self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) - self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + if support.is_emscripten: + # Emscripten timestamps are roundtripped through a 53 bit integer of + # nanoseconds. If we want to represent ~50 years which is an 11 + # digits number of seconds: + # 2*log10(60) + log10(24) + log10(365) + log10(60) + log10(50) + # is about 11. Because 53 * log10(2) is about 16, we only have 5 + # digits worth of sub-second precision. + # Some day it would be good to fix this upstream. + delta=1e-5 + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_atime_ns, atime_ns, delta=1e9 * 1e-5) + self.assertAlmostEqual(st.st_mtime_ns, mtime_ns, delta=1e9 * 1e-5) else: - self.assertEqual(st.st_atime, atime_ns * 1e-9) - self.assertEqual(st.st_mtime, mtime_ns * 1e-9) - self.assertEqual(st.st_atime_ns, atime_ns) - self.assertEqual(st.st_mtime_ns, mtime_ns) + if support_subsecond: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + else: + self.assertEqual(st.st_atime, atime_ns * 1e-9) + self.assertEqual(st.st_mtime, mtime_ns * 1e-9) + self.assertEqual(st.st_atime_ns, atime_ns) + self.assertEqual(st.st_mtime_ns, mtime_ns) def test_utime(self): def set_time(filename, ns): diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index d368f08b610870..92398300f26577 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -116,6 +116,7 @@ def test_clock_monotonic(self): 'need time.pthread_getcpuclockid()') @unittest.skipUnless(hasattr(time, 'clock_gettime'), 'need time.clock_gettime()') + @unittest.skipIf(support.is_emscripten, "Fails to find clock") def test_pthread_getcpuclockid(self): clk_id = time.pthread_getcpuclockid(threading.get_ident()) self.assertTrue(type(clk_id) is int) @@ -539,6 +540,9 @@ def test_perf_counter(self): @unittest.skipIf( support.is_wasi, "process_time not available on WASI" ) + @unittest.skipIf( + support.is_emscripten, "process_time present but doesn't exclude sleep" + ) def test_process_time(self): # process_time() should not include time spend during a sleep start = time.process_time() From c33b6fbf358c1bc14b20e14a1fffff62c6826ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Srinivas=20Reddy=20Thatiparthy=20=28=E0=B0=A4=E0=B0=BE?= =?UTF-8?q?=E0=B0=9F=E0=B0=BF=E0=B0=AA=E0=B0=B0=E0=B1=8D=E0=B0=A4=E0=B0=BF?= =?UTF-8?q?=20=E0=B0=B6=E0=B1=8D=E0=B0=B0=E0=B1=80=E0=B0=A8=E0=B0=BF?= =?UTF-8?q?=E0=B0=B5=E0=B0=BE=E0=B0=B8=E0=B1=8D=20=20=E0=B0=B0=E0=B1=86?= =?UTF-8?q?=E0=B0=A1=E0=B1=8D=E0=B0=A1=E0=B0=BF=29?= Date: Thu, 12 Dec 2024 07:48:12 +0530 Subject: [PATCH 178/427] gh-127740: Add some more tests for earlier PR #127756 (#127818) --- Lib/test/test_bytes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 32cd178fa3b445..7bb1ab38aa4fdf 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -464,6 +464,10 @@ def test_fromhex(self): with self.assertRaises(ValueError) as cm: self.type2test.fromhex(value) self.assertIn("fromhex() arg must contain an even number of hexadecimal digits", str(cm.exception)) + for value, position in (("a ", 1), (" aa a ", 5), (" aa a a ", 5)): + with self.assertRaises(ValueError) as cm: + self.type2test.fromhex(value) + self.assertIn(f"non-hexadecimal number found in fromhex() arg at position {position}", str(cm.exception)) for data, pos in ( # invalid first hexadecimal character From 8bbd379ee30db0320ec3d31c37aee2a503902b0f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 11 Dec 2024 21:24:56 -0600 Subject: [PATCH 179/427] Simplify and speed-up an itertools recipe (gh-127848) --- Doc/library/itertools.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 03966f3d3d694b..3b90d7830f3681 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1015,7 +1015,7 @@ The following recipes have a more mathematical flavor: .. testcode:: def powerset(iterable): - "powerset([1,2,3]) → () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" + # powerset([1,2,3]) → () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3) s = list(iterable) return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) @@ -1104,11 +1104,6 @@ The following recipes have a more mathematical flavor: data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) yield from iter_index(data, 1, start=3) - def is_prime(n): - "Return True if n is prime." - # is_prime(1_000_000_000_000_403) → True - return n > 1 and all(n % p for p in sieve(math.isqrt(n) + 1)) - def factor(n): "Prime factors of n." # factor(99) → 3 3 11 @@ -1123,6 +1118,11 @@ The following recipes have a more mathematical flavor: if n > 1: yield n + def is_prime(n): + "Return True if n is prime." + # is_prime(1_000_000_000_000_403) → True + return n > 1 and next(factor(n)) == n + def totient(n): "Count of natural numbers up to n that are coprime to n." # https://mathworld.wolfram.com/TotientFunction.html From 292afd1d51dd7aacb12a6165f596ae7bb58c9ba8 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 12 Dec 2024 06:49:34 +0000 Subject: [PATCH 180/427] GH-127381: pathlib ABCs: remove remaining uncommon `PathBase` methods (#127714) Remove the following methods from `pathlib._abc.PathBase`: - `expanduser()` - `hardlink_to()` - `touch()` - `chmod()` - `lchmod()` - `owner()` - `group()` - `from_uri()` - `as_uri()` These operations aren't regularly supported in virtual filesystems, so they don't win a place in the `PathBase` interface. (Some of them probably don't deserve a place in `Path` :P.) They're quasi-abstract (except `lchmod()`), and they're not called by other `PathBase` methods. --- Lib/pathlib/_abc.py | 54 ----------------------- Lib/pathlib/_local.py | 28 +++++++++++- Lib/test/test_pathlib/test_pathlib_abc.py | 12 ----- 3 files changed, 27 insertions(+), 67 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 02c6e0500617aa..b10aba85132332 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -571,12 +571,6 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): yield path, dirnames, filenames paths += [path.joinpath(d) for d in reversed(dirnames)] - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - raise UnsupportedOperation(self._unsupported_msg('expanduser()')) - def readlink(self): """ Return the path to which the symbolic link points. @@ -597,20 +591,6 @@ def _symlink_to_target_of(self, link): """ self.symlink_to(link.readlink()) - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - raise UnsupportedOperation(self._unsupported_msg('touch()')) - def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ Create a new directory at this given path. @@ -729,37 +709,3 @@ def move_into(self, target_dir): else: target = self.with_segments(target_dir, name) return self.move(target) - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - raise UnsupportedOperation(self._unsupported_msg('chmod()')) - - def lchmod(self, mode): - """ - Like chmod(), except if the path points to a symlink, the symlink's - permissions are changed, rather than its target's. - """ - self.chmod(mode, follow_symlinks=False) - - def owner(self, *, follow_symlinks=True): - """ - Return the login name of the file owner. - """ - raise UnsupportedOperation(self._unsupported_msg('owner()')) - - def group(self, *, follow_symlinks=True): - """ - Return the group name of the file gid. - """ - raise UnsupportedOperation(self._unsupported_msg('group()')) - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - raise UnsupportedOperation(cls._unsupported_msg('from_uri()')) - - def as_uri(self): - """Return the path as a URI.""" - raise UnsupportedOperation(self._unsupported_msg('as_uri()')) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 85437ec80bfcc4..0dfe9d2390ecff 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -526,7 +526,6 @@ class Path(PathBase, PurePath): but cannot instantiate a WindowsPath on a POSIX system or vice versa. """ __slots__ = () - as_uri = PurePath.as_uri @classmethod def _unsupported_msg(cls, attribute): @@ -813,6 +812,12 @@ def owner(self, *, follow_symlinks=True): """ uid = self.stat(follow_symlinks=follow_symlinks).st_uid return pwd.getpwuid(uid).pw_name + else: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + raise UnsupportedOperation(self._unsupported_msg('owner()')) if grp: def group(self, *, follow_symlinks=True): @@ -821,6 +826,12 @@ def group(self, *, follow_symlinks=True): """ gid = self.stat(follow_symlinks=follow_symlinks).st_gid return grp.getgrgid(gid).gr_name + else: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + raise UnsupportedOperation(self._unsupported_msg('group()')) if hasattr(os, "readlink"): def readlink(self): @@ -892,6 +903,13 @@ def chmod(self, mode, *, follow_symlinks=True): """ os.chmod(self, mode, follow_symlinks=follow_symlinks) + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + def unlink(self, missing_ok=False): """ Remove this file or link. @@ -988,6 +1006,14 @@ def hardlink_to(self, target): Note the order of arguments (self, target) is the reverse of os.link's. """ os.link(target, self) + else: + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) def expanduser(self): """ Return a new path with expanded ~ and ~user constructs diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index f4c364c6fe5109..d770b87dc6a104 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1312,21 +1312,9 @@ def test_unsupported_operation(self): self.assertRaises(e, lambda: list(p.glob('*'))) self.assertRaises(e, lambda: list(p.rglob('*'))) self.assertRaises(e, lambda: list(p.walk())) - self.assertRaises(e, p.expanduser) self.assertRaises(e, p.readlink) self.assertRaises(e, p.symlink_to, 'foo') - self.assertRaises(e, p.hardlink_to, 'foo') self.assertRaises(e, p.mkdir) - self.assertRaises(e, p.touch) - self.assertRaises(e, p.chmod, 0o755) - self.assertRaises(e, p.lchmod, 0o755) - self.assertRaises(e, p.owner) - self.assertRaises(e, p.group) - self.assertRaises(e, p.as_uri) - - def test_as_uri_common(self): - e = UnsupportedOperation - self.assertRaises(e, self.cls('').as_uri) def test_fspath_common(self): self.assertRaises(TypeError, os.fspath, self.cls('')) From 487fdbed40734fd7721457c6f6ffeca03da0b0e7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 12 Dec 2024 11:22:20 +0000 Subject: [PATCH 181/427] GH-125174: Fix compiler warning (GH-127860) Fix compiler warning --- Include/internal/pycore_object.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 22de3c9d4e32ea..668ea47ca727e2 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -184,7 +184,7 @@ PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op); // Makes an immortal object mortal again with the specified refcnt. Should only // be used during runtime finalization. -static inline void _Py_SetMortal(PyObject *op, Py_ssize_t refcnt) +static inline void _Py_SetMortal(PyObject *op, short refcnt) { if (op) { assert(_Py_IsImmortal(op)); From 7146f1894638130940944d4808dae7d144d46227 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 12 Dec 2024 17:39:24 +0000 Subject: [PATCH 182/427] GH-127807: pathlib ABCs: remove `PathBase._unsupported_msg()` (#127855) This method helped us customise the `UnsupportedOperation` message depending on the type. But we're aiming to make `PathBase` a proper ABC soon, so `NotImplementedError` is the right exception to raise there. --- Lib/pathlib/__init__.py | 4 +-- Lib/pathlib/_abc.py | 35 +++++++-------------- Lib/pathlib/_local.py | 37 ++++++++++++++++++----- Lib/test/test_pathlib/test_pathlib.py | 8 +++++ Lib/test/test_pathlib/test_pathlib_abc.py | 12 ++------ 5 files changed, 52 insertions(+), 44 deletions(-) diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 5da3acd31997e5..ec1bac9ef49350 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -5,8 +5,6 @@ operating systems. """ -from pathlib._abc import * from pathlib._local import * -__all__ = (_abc.__all__ + - _local.__all__) +__all__ = _local.__all__ diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index b10aba85132332..b4560295300c28 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -20,15 +20,6 @@ from pathlib._os import copyfileobj -__all__ = ["UnsupportedOperation"] - - -class UnsupportedOperation(NotImplementedError): - """An exception that is raised when an unsupported operation is attempted. - """ - pass - - @functools.cache def _is_case_sensitive(parser): return parser.normcase('Aa') == 'Aa' @@ -353,8 +344,8 @@ class PathBase(PurePathBase): This class provides dummy implementations for many methods that derived classes can override selectively; the default implementations raise - UnsupportedOperation. The most basic methods, such as stat() and open(), - directly raise UnsupportedOperation; these basic methods are called by + NotImplementedError. The most basic methods, such as stat() and open(), + directly raise NotImplementedError; these basic methods are called by other methods such as is_dir() and read_text(). The Path class derives this class to implement local filesystem paths. @@ -363,16 +354,12 @@ class PathBase(PurePathBase): """ __slots__ = () - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported" - def stat(self, *, follow_symlinks=True): """ Return the result of the stat() system call on this path, like os.stat() does. """ - raise UnsupportedOperation(self._unsupported_msg('stat()')) + raise NotImplementedError # Convenience functions for querying the stat results @@ -448,7 +435,7 @@ def open(self, mode='r', buffering=-1, encoding=None, Open the file pointed to by this path and return a file object, as the built-in open() function does. """ - raise UnsupportedOperation(self._unsupported_msg('open()')) + raise NotImplementedError def read_bytes(self): """ @@ -498,7 +485,7 @@ def iterdir(self): The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - raise UnsupportedOperation(self._unsupported_msg('iterdir()')) + raise NotImplementedError def _glob_selector(self, parts, case_sensitive, recurse_symlinks): if case_sensitive is None: @@ -575,14 +562,14 @@ def readlink(self): """ Return the path to which the symbolic link points. """ - raise UnsupportedOperation(self._unsupported_msg('readlink()')) + raise NotImplementedError def symlink_to(self, target, target_is_directory=False): """ Make this path a symlink pointing to the target path. Note the order of arguments (link, target) is the reverse of os.symlink. """ - raise UnsupportedOperation(self._unsupported_msg('symlink_to()')) + raise NotImplementedError def _symlink_to_target_of(self, link): """ @@ -595,7 +582,7 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ Create a new directory at this given path. """ - raise UnsupportedOperation(self._unsupported_msg('mkdir()')) + raise NotImplementedError # Metadata keys supported by this path type. _readable_metadata = _writable_metadata = frozenset() @@ -604,13 +591,13 @@ def _read_metadata(self, keys=None, *, follow_symlinks=True): """ Returns path metadata as a dict with string keys. """ - raise UnsupportedOperation(self._unsupported_msg('_read_metadata()')) + raise NotImplementedError def _write_metadata(self, metadata, *, follow_symlinks=True): """ Sets path metadata from the given dict with string keys. """ - raise UnsupportedOperation(self._unsupported_msg('_write_metadata()')) + raise NotImplementedError def _copy_metadata(self, target, *, follow_symlinks=True): """ @@ -687,7 +674,7 @@ def _delete(self): """ Delete this file or directory (including all sub-directories). """ - raise UnsupportedOperation(self._unsupported_msg('_delete()')) + raise NotImplementedError def move(self, target): """ diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 0dfe9d2390ecff..b933dd512eeb28 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -21,15 +21,22 @@ from pathlib._os import (copyfile, file_metadata_keys, read_file_metadata, write_file_metadata) -from pathlib._abc import UnsupportedOperation, PurePathBase, PathBase +from pathlib._abc import PurePathBase, PathBase __all__ = [ + "UnsupportedOperation", "PurePath", "PurePosixPath", "PureWindowsPath", "Path", "PosixPath", "WindowsPath", ] +class UnsupportedOperation(NotImplementedError): + """An exception that is raised when an unsupported operation is attempted. + """ + pass + + class _PathParents(Sequence): """This object provides sequence-like access to the logical ancestors of a path. Don't try to construct it yourself.""" @@ -527,10 +534,6 @@ class Path(PathBase, PurePath): """ __slots__ = () - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported on this system" - def __new__(cls, *args, **kwargs): if cls is Path: cls = WindowsPath if os.name == 'nt' else PosixPath @@ -817,7 +820,8 @@ def owner(self, *, follow_symlinks=True): """ Return the login name of the file owner. """ - raise UnsupportedOperation(self._unsupported_msg('owner()')) + f = f"{type(self).__name__}.owner()" + raise UnsupportedOperation(f"{f} is unsupported on this system") if grp: def group(self, *, follow_symlinks=True): @@ -831,7 +835,8 @@ def group(self, *, follow_symlinks=True): """ Return the group name of the file gid. """ - raise UnsupportedOperation(self._unsupported_msg('group()')) + f = f"{type(self).__name__}.group()" + raise UnsupportedOperation(f"{f} is unsupported on this system") if hasattr(os, "readlink"): def readlink(self): @@ -839,6 +844,13 @@ def readlink(self): Return the path to which the symbolic link points. """ return self.with_segments(os.readlink(self)) + else: + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + f = f"{type(self).__name__}.readlink()" + raise UnsupportedOperation(f"{f} is unsupported on this system") def touch(self, mode=0o666, exist_ok=True): """ @@ -989,6 +1001,14 @@ def symlink_to(self, target, target_is_directory=False): Note the order of arguments (link, target) is the reverse of os.symlink. """ os.symlink(target, self, target_is_directory) + else: + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + f = f"{type(self).__name__}.symlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") if os.name == 'nt': def _symlink_to_target_of(self, link): @@ -1013,7 +1033,8 @@ def hardlink_to(self, target): Note the order of arguments (self, target) is the reverse of os.link's. """ - raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) + f = f"{type(self).__name__}.hardlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") def expanduser(self): """ Return a new path with expanded ~ and ~user constructs diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index b57ef420bfcbcd..68bff2cf0d511e 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -63,6 +63,14 @@ def needs_symlinks(fn): _tests_needing_symlinks.add(fn.__name__) return fn + + +class UnsupportedOperationTest(unittest.TestCase): + def test_is_notimplemented(self): + self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) + self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) + + # # Tests for the pure classes. # diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index d770b87dc6a104..e230dd188799a5 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -5,7 +5,7 @@ import stat import unittest -from pathlib._abc import UnsupportedOperation, PurePathBase, PathBase +from pathlib._abc import PurePathBase, PathBase from pathlib._types import Parser import posixpath @@ -27,11 +27,6 @@ def needs_windows(fn): return fn -class UnsupportedOperationTest(unittest.TestCase): - def test_is_notimplemented(self): - self.assertTrue(issubclass(UnsupportedOperation, NotImplementedError)) - self.assertTrue(isinstance(UnsupportedOperation(), NotImplementedError)) - # # Tests for the pure classes. # @@ -1294,10 +1289,9 @@ def test_is_absolute_windows(self): class PathBaseTest(PurePathBaseTest): cls = PathBase - def test_unsupported_operation(self): - P = self.cls + def test_not_implemented_error(self): p = self.cls('') - e = UnsupportedOperation + e = NotImplementedError self.assertRaises(e, p.stat) self.assertRaises(e, p.exists) self.assertRaises(e, p.is_dir) From f8dcb8200626a1a06c4a26d8129257f42658a9ff Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 12 Dec 2024 17:59:13 +0000 Subject: [PATCH 183/427] gh-127879: Fix data race in `_PyFreeList_Push` (#127880) Writes to the `ob_tid` field need to use atomics because it may be concurrently read by a non-locking dictionary, list, or structmember read. --- Include/internal/pycore_freelist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index da2d7bf6ae1393..84a5ab30f3eeea 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -51,7 +51,7 @@ static inline int _PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize) { if (fl->size < maxsize && fl->size >= 0) { - *(void **)obj = fl->freelist; + FT_ATOMIC_STORE_PTR_RELAXED(*(void **)obj, fl->freelist); fl->freelist = obj; fl->size++; OBJECT_STAT_INC(to_freelist); From f823910bbd4bf01ec3e1ab7b3cb1d77815138296 Mon Sep 17 00:00:00 2001 From: velemas <10437413+velemas@users.noreply.github.com> Date: Thu, 12 Dec 2024 20:07:55 +0200 Subject: [PATCH 184/427] gh-127865: Fix build failure for systems without thread local support (GH-127866) This PR fixes the build issue introduced by the commit 628f6eb from GH-112207 on systems without thread local support. --- .../Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst | 1 + Python/import.c | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst diff --git a/Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst b/Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst new file mode 100644 index 00000000000000..3fc1d8a1b51d30 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst @@ -0,0 +1 @@ +Fix build failure on systems without thread-locals support. diff --git a/Python/import.c b/Python/import.c index b3c384c27718ce..f3511aaf7b8010 100644 --- a/Python/import.c +++ b/Python/import.c @@ -749,7 +749,7 @@ const char * _PyImport_ResolveNameWithPackageContext(const char *name) { #ifndef HAVE_THREAD_LOCAL - PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK); + PyMutex_Lock(&EXTENSIONS.mutex); #endif if (PKGCONTEXT != NULL) { const char *p = strrchr(PKGCONTEXT, '.'); @@ -759,7 +759,7 @@ _PyImport_ResolveNameWithPackageContext(const char *name) } } #ifndef HAVE_THREAD_LOCAL - PyThread_release_lock(EXTENSIONS.mutex); + PyMutex_Unlock(&EXTENSIONS.mutex); #endif return name; } @@ -768,12 +768,12 @@ const char * _PyImport_SwapPackageContext(const char *newcontext) { #ifndef HAVE_THREAD_LOCAL - PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK); + PyMutex_Lock(&EXTENSIONS.mutex); #endif const char *oldcontext = PKGCONTEXT; PKGCONTEXT = newcontext; #ifndef HAVE_THREAD_LOCAL - PyThread_release_lock(EXTENSIONS.mutex); + PyMutex_Unlock(&EXTENSIONS.mutex); #endif return oldcontext; } From 365451e28368db46ae89a3a990d85c10c2284aa2 Mon Sep 17 00:00:00 2001 From: Andrey Efremov Date: Fri, 13 Dec 2024 03:17:39 +0700 Subject: [PATCH 185/427] gh-127353: Allow to force color output on Windows (#127354) --- Lib/_colorize.py | 17 +++++---- Lib/test/test__colorize.py | 37 +++++++++++++++++++ ...-11-28-15-55-48.gh-issue-127353.i-XOXg.rst | 2 + 3 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 845fb57a90abb8..709081e25ec59b 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -32,14 +32,6 @@ def get_colors(colorize: bool = False) -> ANSIColors: def can_colorize() -> bool: - if sys.platform == "win32": - try: - import nt - - if not nt._supports_virtual_terminal(): - return False - except (ImportError, AttributeError): - return False if not sys.flags.ignore_environment: if os.environ.get("PYTHON_COLORS") == "0": return False @@ -58,6 +50,15 @@ def can_colorize() -> bool: if not hasattr(sys.stderr, "fileno"): return False + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + try: return os.isatty(sys.stderr.fileno()) except io.UnsupportedOperation: diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index d55b97ade68cef..7a65d63f49eed7 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -50,10 +50,47 @@ def test_colorized_detection_checks_for_environment_variables(self): with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {}): + self.assertEqual(_colorize.can_colorize(), True) + isatty_mock.return_value = False with unittest.mock.patch("os.environ", {}): self.assertEqual(_colorize.can_colorize(), False) + @force_not_colorized + @unittest.skipUnless(sys.platform == "win32", "Windows only") + def test_colorized_detection_checks_for_environment_variables_no_vt(self): + with (unittest.mock.patch("nt._supports_virtual_terminal", return_value=False), + unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.flags", unittest.mock.MagicMock(ignore_environment=False)), + unittest.mock.patch("_colorize.can_colorize", ORIGINAL_CAN_COLORIZE)): + isatty_mock.return_value = True + with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {}): + self.assertEqual(_colorize.can_colorize(), False) + + isatty_mock.return_value = False + with unittest.mock.patch("os.environ", {}): + self.assertEqual(_colorize.can_colorize(), False) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst b/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst new file mode 100644 index 00000000000000..88661b9a611071 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst @@ -0,0 +1,2 @@ +Allow to force color output on Windows using environment variables. Patch by +Andrey Efremov. From ed037d229f64db90aea00f397e9ce1b2f4a22d3f Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 12 Dec 2024 20:27:29 +0000 Subject: [PATCH 186/427] Fix typos in `Lib/_pydecimal.py` (#127700) --- Lib/_pydecimal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 5b60570c6c592a..ec036199331396 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -97,7 +97,7 @@ class DecimalException(ArithmeticError): Used exceptions derive from this. If an exception derives from another exception besides this (such as - Underflow (Inexact, Rounded, Subnormal) that indicates that it is only + Underflow (Inexact, Rounded, Subnormal)) that indicates that it is only called if the others are present. This isn't actually used for anything, though. @@ -145,7 +145,7 @@ class InvalidOperation(DecimalException): x ** (+-)INF An operand is invalid - The result of the operation after these is a quiet positive NaN, + The result of the operation after this is a quiet positive NaN, except when the cause is a signaling NaN, in which case the result is also a quiet NaN, but with the original sign, and an optional diagnostic information. From a8ffe661548e16ad02dbe6cb8a89513d7ed2a42c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 12 Dec 2024 23:11:20 +0200 Subject: [PATCH 187/427] Revert "gh-127353: Allow to force color output on Windows (#127354)" (#127889) This reverts commit 365451e28368db46ae89a3a990d85c10c2284aa2. --- Lib/_colorize.py | 17 ++++----- Lib/test/test__colorize.py | 37 ------------------- ...-11-28-15-55-48.gh-issue-127353.i-XOXg.rst | 2 - 3 files changed, 8 insertions(+), 48 deletions(-) delete mode 100644 Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 709081e25ec59b..845fb57a90abb8 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -32,6 +32,14 @@ def get_colors(colorize: bool = False) -> ANSIColors: def can_colorize() -> bool: + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False if not sys.flags.ignore_environment: if os.environ.get("PYTHON_COLORS") == "0": return False @@ -50,15 +58,6 @@ def can_colorize() -> bool: if not hasattr(sys.stderr, "fileno"): return False - if sys.platform == "win32": - try: - import nt - - if not nt._supports_virtual_terminal(): - return False - except (ImportError, AttributeError): - return False - try: return os.isatty(sys.stderr.fileno()) except io.UnsupportedOperation: diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index 7a65d63f49eed7..d55b97ade68cef 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -50,47 +50,10 @@ def test_colorized_detection_checks_for_environment_variables(self): with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): self.assertEqual(_colorize.can_colorize(), False) - with unittest.mock.patch("os.environ", {}): - self.assertEqual(_colorize.can_colorize(), True) - isatty_mock.return_value = False with unittest.mock.patch("os.environ", {}): self.assertEqual(_colorize.can_colorize(), False) - @force_not_colorized - @unittest.skipUnless(sys.platform == "win32", "Windows only") - def test_colorized_detection_checks_for_environment_variables_no_vt(self): - with (unittest.mock.patch("nt._supports_virtual_terminal", return_value=False), - unittest.mock.patch("os.isatty") as isatty_mock, - unittest.mock.patch("sys.flags", unittest.mock.MagicMock(ignore_environment=False)), - unittest.mock.patch("_colorize.can_colorize", ORIGINAL_CAN_COLORIZE)): - isatty_mock.return_value = True - with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): - self.assertEqual(_colorize.can_colorize(), False) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): - self.assertEqual(_colorize.can_colorize(), True) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): - self.assertEqual(_colorize.can_colorize(), False) - with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): - self.assertEqual(_colorize.can_colorize(), False) - with unittest.mock.patch("os.environ", - {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): - self.assertEqual(_colorize.can_colorize(), True) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): - self.assertEqual(_colorize.can_colorize(), True) - with unittest.mock.patch("os.environ", - {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): - self.assertEqual(_colorize.can_colorize(), False) - with unittest.mock.patch("os.environ", - {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): - self.assertEqual(_colorize.can_colorize(), False) - with unittest.mock.patch("os.environ", {}): - self.assertEqual(_colorize.can_colorize(), False) - - isatty_mock.return_value = False - with unittest.mock.patch("os.environ", {}): - self.assertEqual(_colorize.can_colorize(), False) - if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst b/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst deleted file mode 100644 index 88661b9a611071..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow to force color output on Windows using environment variables. Patch by -Andrey Efremov. From 8ac307f0d6834148471d2e12a45bf022e659164c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Thu, 12 Dec 2024 21:41:46 +0000 Subject: [PATCH 188/427] GH-127724: don't use sysconfig to calculate the venv local include path (#127731) --- Lib/venv/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index ca1af84e6705fe..dc4c9ef3531991 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -103,8 +103,6 @@ def _venv_path(self, env_dir, name): vars = { 'base': env_dir, 'platbase': env_dir, - 'installed_base': env_dir, - 'installed_platbase': env_dir, } return sysconfig.get_path(name, scheme='venv', vars=vars) @@ -175,9 +173,20 @@ def create_if_needed(d): context.python_dir = dirname context.python_exe = exename binpath = self._venv_path(env_dir, 'scripts') - incpath = self._venv_path(env_dir, 'include') libpath = self._venv_path(env_dir, 'purelib') + # PEP 405 says venvs should create a local include directory. + # See https://peps.python.org/pep-0405/#include-files + # XXX: This directory is not exposed in sysconfig or anywhere else, and + # doesn't seem to be utilized by modern packaging tools. We keep it + # for backwards-compatibility, and to follow the PEP, but I would + # recommend against using it, as most tooling does not pass it to + # compilers. Instead, until we standardize a site-specific include + # directory, I would recommend installing headers as package data, + # and providing some sort of API to get the include directories. + # Example: https://numpy.org/doc/2.1/reference/generated/numpy.get_include.html + incpath = os.path.join(env_dir, 'Include' if os.name == 'nt' else 'include') + context.inc_path = incpath create_if_needed(incpath) context.lib_path = libpath From 0cbc19d59e409854f2b9bdda75e1af2b6cd89ac2 Mon Sep 17 00:00:00 2001 From: Daniel Haag <121057143+denialhaag@users.noreply.github.com> Date: Thu, 12 Dec 2024 22:43:44 +0100 Subject: [PATCH 189/427] Fix typo in traceback docs (#127884) --- Doc/library/traceback.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 4899ed64ebad8d..b0ee3fc56ad735 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -274,7 +274,7 @@ Module-Level Functions :class:`!TracebackException` objects are created from actual exceptions to capture data for later printing. They offer a more lightweight method of storing this information by avoiding holding references to -:ref:`traceback` and :ref:`frame` objects +:ref:`traceback` and :ref:`frame` objects. In addition, they expose more options to configure the output compared to the module-level functions described above. From ba2d2fda93a03a91ac6cdff319fd23ef51848d51 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 13 Dec 2024 05:49:02 +0800 Subject: [PATCH 190/427] gh-127845: Minor improvements to iOS test runner script (#127846) Uses symlinks to install iOS framework into testbed clone, adds a verbose mode to the iOS runner to hide most Xcode output, adds another mechanism to disable terminal colors, and ensures that stdout is flushed after every write. --- Makefile.pre.in | 2 +- iOS/testbed/__main__.py | 66 ++++++++++++++----- iOS/testbed/iOSTestbedTests/iOSTestbedTests.m | 5 +- 3 files changed, 53 insertions(+), 20 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 7b66802147dc3a..3e880f7800fccf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2169,7 +2169,7 @@ testios: $(PYTHON_FOR_BUILD) $(srcdir)/iOS/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" # Run the testbed project - $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run -- test -uall --single-process --rerun -W + $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run --verbose -- test -uall --single-process --rerun -W # Like test, but using --slow-ci which enables all test resources and use # longer timeout. Run an optional pybuildbot.identify script to include diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py index 22570ee0f3ed04..068272835a5b95 100644 --- a/iOS/testbed/__main__.py +++ b/iOS/testbed/__main__.py @@ -141,10 +141,12 @@ async def log_stream_task(initial_devices): else: suppress_dupes = False sys.stdout.write(line) + sys.stdout.flush() -async def xcode_test(location, simulator): +async def xcode_test(location, simulator, verbose): # Run the test suite on the named simulator + print("Starting xcodebuild...") args = [ "xcodebuild", "test", @@ -159,6 +161,9 @@ async def xcode_test(location, simulator): "-derivedDataPath", str(location / "DerivedData"), ] + if not verbose: + args += ["-quiet"] + async with async_process( *args, stdout=subprocess.PIPE, @@ -166,6 +171,7 @@ async def xcode_test(location, simulator): ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): sys.stdout.write(line) + sys.stdout.flush() status = await asyncio.wait_for(process.wait(), timeout=1) exit(status) @@ -182,7 +188,9 @@ def clone_testbed( sys.exit(10) if framework is None: - if not (source / "Python.xcframework/ios-arm64_x86_64-simulator/bin").is_dir(): + if not ( + source / "Python.xcframework/ios-arm64_x86_64-simulator/bin" + ).is_dir(): print( f"The testbed being cloned ({source}) does not contain " f"a simulator framework. Re-run with --framework" @@ -202,33 +210,48 @@ def clone_testbed( ) sys.exit(13) - print("Cloning testbed project...") - shutil.copytree(source, target) + print("Cloning testbed project:") + print(f" Cloning {source}...", end="", flush=True) + shutil.copytree(source, target, symlinks=True) + print(" done") if framework is not None: if framework.suffix == ".xcframework": - print("Installing XCFramework...") - xc_framework_path = target / "Python.xcframework" - shutil.rmtree(xc_framework_path) - shutil.copytree(framework, xc_framework_path) + print(" Installing XCFramework...", end="", flush=True) + xc_framework_path = (target / "Python.xcframework").resolve() + if xc_framework_path.is_dir(): + shutil.rmtree(xc_framework_path) + else: + xc_framework_path.unlink() + xc_framework_path.symlink_to( + framework.relative_to(xc_framework_path.parent, walk_up=True) + ) + print(" done") else: - print("Installing simulator Framework...") + print(" Installing simulator framework...", end="", flush=True) sim_framework_path = ( target / "Python.xcframework" / "ios-arm64_x86_64-simulator" + ).resolve() + if sim_framework_path.is_dir(): + shutil.rmtree(sim_framework_path) + else: + sim_framework_path.unlink() + sim_framework_path.symlink_to( + framework.relative_to(sim_framework_path.parent, walk_up=True) ) - shutil.rmtree(sim_framework_path) - shutil.copytree(framework, sim_framework_path) + print(" done") else: - print("Using pre-existing iOS framework.") + print(" Using pre-existing iOS framework.") for app_src in apps: - print(f"Installing app {app_src.name!r}...") + print(f" Installing app {app_src.name!r}...", end="", flush=True) app_target = target / f"iOSTestbed/app/{app_src.name}" if app_target.is_dir(): shutil.rmtree(app_target) shutil.copytree(app_src, app_target) + print(" done") - print(f"Testbed project created in {target}") + print(f"Successfully cloned testbed: {target.resolve()}") def update_plist(testbed_path, args): @@ -243,10 +266,11 @@ def update_plist(testbed_path, args): plistlib.dump(info, f) -async def run_testbed(simulator: str, args: list[str]): +async def run_testbed(simulator: str, args: list[str], verbose: bool=False): location = Path(__file__).parent - print("Updating plist...") + print("Updating plist...", end="", flush=True) update_plist(location, args) + print(" done.") # Get the list of devices that are booted at the start of the test run. # The simulator started by the test suite will be detected as the new @@ -256,7 +280,7 @@ async def run_testbed(simulator: str, args: list[str]): try: async with asyncio.TaskGroup() as tg: tg.create_task(log_stream_task(initial_devices)) - tg.create_task(xcode_test(location, simulator)) + tg.create_task(xcode_test(location, simulator=simulator, verbose=verbose)) except* MySystemExit as e: raise SystemExit(*e.exceptions[0].args) from None except* subprocess.CalledProcessError as e: @@ -315,6 +339,11 @@ def main(): default="iPhone SE (3rd Generation)", help="The name of the simulator to use (default: 'iPhone SE (3rd Generation)')", ) + run.add_argument( + "-v", "--verbose", + action="store_true", + help="Enable verbose output", + ) try: pos = sys.argv.index("--") @@ -330,7 +359,7 @@ def main(): clone_testbed( source=Path(__file__).parent, target=Path(context.location), - framework=Path(context.framework) if context.framework else None, + framework=Path(context.framework).resolve() if context.framework else None, apps=[Path(app) for app in context.apps], ) elif context.subcommand == "run": @@ -348,6 +377,7 @@ def main(): asyncio.run( run_testbed( simulator=context.simulator, + verbose=context.verbose, args=test_args, ) ) diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m index ac78456a61e65e..6db38253396c8d 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -24,8 +24,11 @@ - (void)testPython { NSString *resourcePath = [[NSBundle mainBundle] resourcePath]; - // Disable all color, as the Xcode log can't display color + // Set some other common environment indicators to disable color, as the + // Xcode log can't display color. Stdout will report that it is *not* a + // TTY. setenv("NO_COLOR", "1", true); + setenv("PY_COLORS", "0", true); // Arguments to pass into the test suite runner. // argv[0] must identify the process; any subsequent arg From 58942a07df8811afba9c58dc16c1aab244ccf27a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 13 Dec 2024 10:26:22 +0100 Subject: [PATCH 191/427] Document PyObject_SelfIter (#127861) --- Doc/c-api/object.rst | 6 ++++++ Doc/data/refcounts.dat | 3 +++ 2 files changed, 9 insertions(+) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 1ae3c46bea46ea..f97ade01e67850 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -509,6 +509,12 @@ Object Protocol iterated. +.. c:function:: PyObject* PyObject_SelfIter(PyObject *obj) + + This is equivalent to the Python ``__iter__(self): return self`` method. + It is intended for :term:`iterator` types, to be used in the :c:member:`PyTypeObject.tp_iter` slot. + + .. c:function:: PyObject* PyObject_GetAIter(PyObject *o) This is the equivalent to the Python expression ``aiter(o)``. Takes an diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 3f49c88c3cc028..a043af48ba7a05 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1849,6 +1849,9 @@ PyObject_RichCompareBool:PyObject*:o1:0: PyObject_RichCompareBool:PyObject*:o2:0: PyObject_RichCompareBool:int:opid:: +PyObject_SelfIter:PyObject*::+1: +PyObject_SelfIter:PyObject*:obj:0: + PyObject_SetAttr:int::: PyObject_SetAttr:PyObject*:o:0: PyObject_SetAttr:PyObject*:attr_name:0: From 11ff3286b7e821bf439bc7caa0fa712e3bc3846a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20K=C3=A1lm=C3=A1n?= Date: Fri, 13 Dec 2024 10:27:02 +0100 Subject: [PATCH 192/427] link to the correct output method in documentation (#127857) --- Doc/library/http.cookies.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index 4ce2e3c4f4cb42..ad37a0fca4742d 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -98,7 +98,7 @@ Cookie Objects .. method:: BaseCookie.output(attrs=None, header='Set-Cookie:', sep='\r\n') Return a string representation suitable to be sent as HTTP headers. *attrs* and - *header* are sent to each :class:`Morsel`'s :meth:`output` method. *sep* is used + *header* are sent to each :class:`Morsel`'s :meth:`~Morsel.output` method. *sep* is used to join the headers together, and is by default the combination ``'\r\n'`` (CRLF). From 9b4bbf4401291636e5db90511a0548fffb23a505 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 13 Dec 2024 09:54:59 +0000 Subject: [PATCH 193/427] GH-125174: Don't use `UINT32_MAX` in header file (GH-127863) --- Include/refcount.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/refcount.h b/Include/refcount.h index 6908c426141378..d98b2dfcf37202 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -377,7 +377,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) #if SIZEOF_VOID_P > 4 /* If an object has been freed, it will have a negative full refcnt * If it has not it been freed, will have a very large refcnt */ - if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (UINT32_MAX - (1<<20))) { + if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((PY_UINT32_T)-1) - (1<<20))) { #else if (op->ob_refcnt <= 0) { #endif From 5fc6bb2754a25157575efc0b37da78c629fea46e Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 13 Dec 2024 11:06:26 +0100 Subject: [PATCH 194/427] gh-126868: Add freelist for compact int objects (GH-126865) --- Include/internal/pycore_freelist_state.h | 2 + Include/internal/pycore_long.h | 2 + ...-11-16-22-37-46.gh-issue-126868.yOoHSY.rst | 1 + Objects/longobject.c | 78 ++++++++++++++----- Objects/object.c | 1 + Python/bytecodes.c | 25 +++--- Python/executor_cases.c.h | 24 +++--- Python/generated_cases.c.h | 24 +++--- 8 files changed, 102 insertions(+), 55 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 4e04cf431e0b31..a1a94c1f2dc880 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -14,6 +14,7 @@ extern "C" { # define Py_dicts_MAXFREELIST 80 # define Py_dictkeys_MAXFREELIST 80 # define Py_floats_MAXFREELIST 100 +# define Py_ints_MAXFREELIST 100 # define Py_slices_MAXFREELIST 1 # define Py_contexts_MAXFREELIST 255 # define Py_async_gens_MAXFREELIST 80 @@ -35,6 +36,7 @@ struct _Py_freelist { struct _Py_freelists { struct _Py_freelist floats; + struct _Py_freelist ints; struct _Py_freelist tuples[PyTuple_MAXSAVESIZE]; struct _Py_freelist lists; struct _Py_freelist dicts; diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 196b4152280a35..8bead00e70640c 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -55,6 +55,8 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); /* other API */ +PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self); + #define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints) // _PyLong_GetZero() and _PyLong_GetOne() must always be available diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst new file mode 100644 index 00000000000000..fd1570908c1fd6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst @@ -0,0 +1 @@ +Increase performance of :class:`int` by adding a freelist for compact ints. diff --git a/Objects/longobject.c b/Objects/longobject.c index 4aa35685b509f2..96d59f542a7c3c 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6,6 +6,7 @@ #include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_call.h" // _PyObject_MakeTpCall +#include "pycore_freelist.h" // _Py_FREELIST_FREE, _Py_FREELIST_POP #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS @@ -42,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED((PyObject *)op, _PyLong_ExactDealloc); } static inline int @@ -220,15 +221,18 @@ _PyLong_FromMedium(sdigit x) { assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - /* We could use a freelist here */ - PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject)); + + PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); if (v == NULL) { - PyErr_NoMemory(); - return NULL; + v = PyObject_Malloc(sizeof(PyLongObject)); + if (v == NULL) { + PyErr_NoMemory(); + return NULL; + } + _PyObject_Init((PyObject*)v, &PyLong_Type); } digit abs_x = x < 0 ? -x : x; _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); - _PyObject_Init((PyObject*)v, &PyLong_Type); v->long_value.ob_digit[0] = abs_x; return (PyObject*)v; } @@ -3611,24 +3615,60 @@ long_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_RICHCOMPARE(result, 0, op); } +static inline int +compact_int_is_small(PyObject *self) +{ + PyLongObject *pylong = (PyLongObject *)self; + assert(_PyLong_IsCompact(pylong)); + stwodigits ival = medium_value(pylong); + if (IS_SMALL_INT(ival)) { + PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); + if (pylong == small_pylong) { + return 1; + } + } + return 0; +} + +void +_PyLong_ExactDealloc(PyObject *self) +{ + assert(PyLong_CheckExact(self)); + if (_PyLong_IsCompact((PyLongObject *)self)) { + #ifndef Py_GIL_DISABLED + if (compact_int_is_small(self)) { + // See PEP 683, section Accidental De-Immortalizing for details + _Py_SetImmortal(self); + return; + } + #endif + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; + } + PyObject_Free(self); +} + static void long_dealloc(PyObject *self) { - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref small Ints out of existence. Instead, - * since small Ints are immortal, re-set the reference count. - */ - PyLongObject *pylong = (PyLongObject*)self; - if (pylong && _PyLong_IsCompact(pylong)) { - stwodigits ival = medium_value(pylong); - if (IS_SMALL_INT(ival)) { - PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); - if (pylong == small_pylong) { - _Py_SetImmortal(self); - return; - } + assert(self); + if (_PyLong_IsCompact((PyLongObject *)self)) { + if (compact_int_is_small(self)) { + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref small Ints out of existence. Instead, + * since small Ints are immortal, re-set the reference count. + * + * See PEP 683, section Accidental De-Immortalizing for details + */ + _Py_SetImmortal(self); + return; + } + if (PyLong_CheckExact(self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; } } + Py_TYPE(self)->tp_free(self); } diff --git a/Objects/object.c b/Objects/object.c index c64675b5e1d6c2..d584414c559b9d 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -936,6 +936,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree); } clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free); + clear_freelist(&freelists->ints, is_finalization, free_object); } /* diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d0e4c2bc45489b..f0eb5405faeff5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -26,6 +26,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject +#include "pycore_long.h" // _PyLong_ExactDealloc() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_tuple.h" // _PyTuple_ITEMS() @@ -514,8 +515,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -527,8 +528,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -540,8 +541,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -801,7 +802,7 @@ dummy_func( assert(res_o != NULL); Py_INCREF(res_o); #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -821,7 +822,7 @@ dummy_func( DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -842,7 +843,7 @@ dummy_func( PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -959,7 +960,7 @@ dummy_func( assert(old_value != NULL); UNLOCK_OBJECT(list); // unlock before decrefs! Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); } @@ -2476,9 +2477,9 @@ dummy_func( Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); DEAD(left); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); DEAD(right); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 18f19773d25c90..19ba67a8af6769 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -626,8 +626,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -646,8 +646,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -666,8 +666,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1000,7 +1000,7 @@ assert(res_o != NULL); Py_INCREF(res_o); #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1042,7 +1042,7 @@ } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1081,7 +1081,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1264,7 +1264,7 @@ assert(old_value != NULL); UNLOCK_OBJECT(list); // unlock before decrefs! Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -3075,8 +3075,8 @@ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index fc0f55555f5c36..51227c9868b8cc 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -118,8 +118,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -285,8 +285,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -356,8 +356,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -590,7 +590,7 @@ assert(res_o != NULL); Py_INCREF(res_o); #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -622,7 +622,7 @@ DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -654,7 +654,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -3333,8 +3333,8 @@ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } @@ -7721,7 +7721,7 @@ assert(old_value != NULL); UNLOCK_OBJECT(list); // unlock before decrefs! Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); From e62e1ca4553dbcf9d7f89be24bebcbd9213f9ae5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 13 Dec 2024 11:00:00 +0000 Subject: [PATCH 195/427] GH-126833: Dumps graphviz representation of executor graph. (GH-126880) --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_optimizer.h | 5 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 4 + Python/ceval.c | 1 + Python/clinic/sysmodule.c.h | 58 +++++++- Python/optimizer.c | 136 +++++++++++++++++- Python/sysmodule.c | 25 ++++ 9 files changed, 230 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index c12e242d560bde..90214a314031d1 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1129,6 +1129,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(origin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(out_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outpath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(owner)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pages)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index dfd9f2b799ec8e..97a75d0c46c867 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -618,6 +618,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(origin) STRUCT_FOR_ID(out_fd) STRUCT_FOR_ID(outgoing) + STRUCT_FOR_ID(outpath) STRUCT_FOR_ID(overlapped) STRUCT_FOR_ID(owner) STRUCT_FOR_ID(pages) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 6d70b42f708854..bc7cfcde613d65 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -60,6 +60,9 @@ typedef struct { }; uint64_t operand0; // A cache entry uint64_t operand1; +#ifdef Py_STATS + uint64_t execution_count; +#endif } _PyUOpInstruction; typedef struct { @@ -285,6 +288,8 @@ static inline int is_terminator(const _PyUOpInstruction *uop) ); } +PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index b631382cae058a..4f928cc050bf8e 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1127,6 +1127,7 @@ extern "C" { INIT_ID(origin), \ INIT_ID(out_fd), \ INIT_ID(outgoing), \ + INIT_ID(outpath), \ INIT_ID(overlapped), \ INIT_ID(owner), \ INIT_ID(pages), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 24cec3a4fded7a..5b78d038fc1192 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -2268,6 +2268,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(outpath); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(overlapped); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Python/ceval.c b/Python/ceval.c index 5eda033eced628..fd891d7839151e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1095,6 +1095,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int UOP_PAIR_INC(uopcode, lastuop); #ifdef Py_STATS trace_uop_execution_counter++; + ((_PyUOpInstruction *)next_uop)[-1].execution_count++; #endif switch (uopcode) { diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 86c42ceffc5e31..cfcbd55388efa0 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1481,6 +1481,62 @@ sys_is_stack_trampoline_active(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys_is_stack_trampoline_active_impl(module); } +PyDoc_STRVAR(sys__dump_tracelets__doc__, +"_dump_tracelets($module, /, outpath)\n" +"--\n" +"\n" +"Dump the graph of tracelets in graphviz format"); + +#define SYS__DUMP_TRACELETS_METHODDEF \ + {"_dump_tracelets", _PyCFunction_CAST(sys__dump_tracelets), METH_FASTCALL|METH_KEYWORDS, sys__dump_tracelets__doc__}, + +static PyObject * +sys__dump_tracelets_impl(PyObject *module, PyObject *outpath); + +static PyObject * +sys__dump_tracelets(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(outpath), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"outpath", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_dump_tracelets", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *outpath; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + outpath = args[0]; + return_value = sys__dump_tracelets_impl(module, outpath); + +exit: + return return_value; +} + PyDoc_STRVAR(sys__getframemodulename__doc__, "_getframemodulename($module, /, depth=0)\n" "--\n" @@ -1668,4 +1724,4 @@ sys__is_gil_enabled(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=6d4f6cd20419b675 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=568b0a0069dc43e8 input=a9049054013a1b77]*/ diff --git a/Python/optimizer.c b/Python/optimizer.c index 6a232218981dcd..6a4d20fad76c15 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,6 +1,7 @@ +#include "Python.h" + #ifdef _Py_TIER2 -#include "Python.h" #include "opcode.h" #include "pycore_interp.h" #include "pycore_backoff.h" @@ -474,6 +475,9 @@ add_to_trace( trace[trace_length].target = target; trace[trace_length].oparg = oparg; trace[trace_length].operand0 = operand; +#ifdef Py_STATS + trace[trace_length].execution_count = 0; +#endif return trace_length + 1; } @@ -983,6 +987,9 @@ static void make_exit(_PyUOpInstruction *inst, int opcode, int target) inst->operand0 = 0; inst->format = UOP_FORMAT_TARGET; inst->target = target; +#ifdef Py_STATS + inst->execution_count = 0; +#endif } /* Convert implicit exits, errors and deopts @@ -1709,4 +1716,131 @@ _Py_Executors_InvalidateCold(PyInterpreterState *interp) _Py_Executors_InvalidateAll(interp, 0); } +static void +write_str(PyObject *str, FILE *out) +{ + // Encode the Unicode object to the specified encoding + PyObject *encoded_obj = PyUnicode_AsEncodedString(str, "utf8", "strict"); + if (encoded_obj == NULL) { + PyErr_Clear(); + return; + } + const char *encoded_str = PyBytes_AsString(encoded_obj); + Py_ssize_t encoded_size = PyBytes_Size(encoded_obj); + fwrite(encoded_str, 1, encoded_size, out); + Py_DECREF(encoded_obj); +} + +static int +find_line_number(PyCodeObject *code, _PyExecutorObject *executor) +{ + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->op.code; + if (opcode == ENTER_EXECUTOR) { + _PyExecutorObject *exec = code->co_executors->executors[instr->op.arg]; + if (exec == executor) { + return PyCode_Addr2Line(code, i*2); + } + } + i += _PyOpcode_Caches[_Py_GetBaseCodeUnit(code, i).op.code]; + } + return -1; +} + +/* Writes the node and outgoing edges for a single tracelet in graphviz format. + * Each tracelet is presented as a table of the uops it contains. + * If Py_STATS is enabled, execution counts are included. + * + * https://graphviz.readthedocs.io/en/stable/manual.html + * https://graphviz.org/gallery/ + */ +static void +executor_to_gv(_PyExecutorObject *executor, FILE *out) +{ + PyCodeObject *code = executor->vm_data.code; + fprintf(out, "executor_%p [\n", executor); + fprintf(out, " shape = none\n"); + + /* Write the HTML table for the uops */ + fprintf(out, " label = <\n"); + fprintf(out, " \n"); + if (code == NULL) { + fprintf(out, " \n"); + } + else { + fprintf(out, " \n", line); + } + for (uint32_t i = 0; i < executor->code_size; i++) { + /* Write row for uop. + * The `port` is a marker so that outgoing edges can + * be placed correctly. If a row is marked `port=17`, + * then the outgoing edge is `{EXEC_NAME}:17 -> {TARGET}` + * https://graphviz.readthedocs.io/en/stable/manual.html#node-ports-compass + */ + _PyUOpInstruction const *inst = &executor->trace[i]; + const char *opname = _PyOpcode_uop_name[inst->opcode]; +#ifdef Py_STATS + fprintf(out, " \n", i, opname, inst->execution_count); +#else + fprintf(out, " \n", i, opname); +#endif + if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { + break; + } + } + fprintf(out, "
Executor
No code object
"); + write_str(code->co_qualname, out); + int line = find_line_number(code, executor); + fprintf(out, ": %d
%s -- %" PRIu64 "
%s
>\n"); + fprintf(out, "]\n\n"); + + /* Write all the outgoing edges */ + for (uint32_t i = 0; i < executor->code_size; i++) { + _PyUOpInstruction const *inst = &executor->trace[i]; + uint16_t flags = _PyUop_Flags[inst->opcode]; + _PyExitData *exit = NULL; + if (inst->opcode == _EXIT_TRACE) { + exit = (_PyExitData *)inst->operand0; + } + else if (flags & HAS_EXIT_FLAG) { + assert(inst->format == UOP_FORMAT_JUMP); + _PyUOpInstruction const *exit_inst = &executor->trace[inst->jump_target]; + assert(exit_inst->opcode == _EXIT_TRACE); + exit = (_PyExitData *)exit_inst->operand0; + } + if (exit != NULL && exit->executor != NULL) { + fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); + } + if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { + break; + } + } +} + +/* Write the graph of all the live tracelets in graphviz format. */ +int +_PyDumpExecutors(FILE *out) +{ + fprintf(out, "digraph ideal {\n\n"); + fprintf(out, " rankdir = \"LR\"\n\n"); + PyInterpreterState *interp = PyInterpreterState_Get(); + for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { + executor_to_gv(exec, out); + exec = exec->vm_data.links.next; + } + fprintf(out, "}\n\n"); + return 0; +} + +#else + +int +_PyDumpExecutors(FILE *out) +{ + PyErr_SetString(PyExc_NotImplementedError, "No JIT available"); + return -1; +} + #endif /* _Py_TIER2 */ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6df297f364c5d3..d6719f9bb0af91 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2344,6 +2344,30 @@ sys_is_stack_trampoline_active_impl(PyObject *module) Py_RETURN_FALSE; } +/*[clinic input] +sys._dump_tracelets + + outpath: object + +Dump the graph of tracelets in graphviz format +[clinic start generated code]*/ + +static PyObject * +sys__dump_tracelets_impl(PyObject *module, PyObject *outpath) +/*[clinic end generated code: output=a7fe265e2bc3b674 input=5bff6880cd28ffd1]*/ +{ + FILE *out = _Py_fopen_obj(outpath, "wb"); + if (out == NULL) { + return NULL; + } + int err = _PyDumpExecutors(out); + fclose(out); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] sys._getframemodulename @@ -2603,6 +2627,7 @@ static PyMethodDef sys_methods[] = { #endif SYS__GET_CPU_COUNT_CONFIG_METHODDEF SYS__IS_GIL_ENABLED_METHODDEF + SYS__DUMP_TRACELETS_METHODDEF {NULL, NULL} // sentinel }; From 6ff38fc4e2af8e795dc791be6ea596d2146d4119 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 13 Dec 2024 13:53:47 +0100 Subject: [PATCH 196/427] gh-127870: Detect recursive calls in ctypes _as_parameter_ handling (#127872) --- Lib/test/test_ctypes/test_as_parameter.py | 12 ++++++++-- ...-12-12-16-59-42.gh-issue-127870._NFG-3.rst | 2 ++ Modules/_ctypes/_ctypes.c | 22 ++++++++++++++++++- 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-12-16-59-42.gh-issue-127870._NFG-3.rst diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index cc62b1a22a3b06..c5e1840b0eb7af 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -198,8 +198,16 @@ class A: a = A() a._as_parameter_ = a - with self.assertRaises(RecursionError): - c_int.from_param(a) + for c_type in ( + ctypes.c_wchar_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_int, # PyCSimpleType + POINT, # CDataType + ): + with self.subTest(c_type=c_type): + with self.assertRaises(RecursionError): + c_type.from_param(a) class AsParamWrapper: diff --git a/Misc/NEWS.d/next/Library/2024-12-12-16-59-42.gh-issue-127870._NFG-3.rst b/Misc/NEWS.d/next/Library/2024-12-12-16-59-42.gh-issue-127870._NFG-3.rst new file mode 100644 index 00000000000000..99b2df00032082 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-12-16-59-42.gh-issue-127870._NFG-3.rst @@ -0,0 +1,2 @@ +Detect recursive calls in ctypes ``_as_parameter_`` handling. +Patch by Victor Stinner. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 34529bce496d88..bb4699884057ba 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1052,8 +1052,13 @@ CDataType_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) return NULL; } if (as_parameter) { + if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { + Py_DECREF(as_parameter); + return NULL; + } value = CDataType_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); + _Py_LeaveRecursiveCall(); return value; } PyErr_Format(PyExc_TypeError, @@ -1843,8 +1848,13 @@ c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) return NULL; } if (as_parameter) { + if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { + Py_DECREF(as_parameter); + return NULL; + } value = c_wchar_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); + _Py_LeaveRecursiveCall(); return value; } PyErr_Format(PyExc_TypeError, @@ -1927,8 +1937,13 @@ c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) return NULL; } if (as_parameter) { + if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { + Py_DECREF(as_parameter); + return NULL; + } value = c_char_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); + _Py_LeaveRecursiveCall(); return value; } PyErr_Format(PyExc_TypeError, @@ -2079,8 +2094,13 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) return NULL; } if (as_parameter) { + if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { + Py_DECREF(as_parameter); + return NULL; + } value = c_void_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); + _Py_LeaveRecursiveCall(); return value; } PyErr_Format(PyExc_TypeError, @@ -2447,9 +2467,9 @@ PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, return NULL; } value = PyCSimpleType_from_param_impl(type, cls, as_parameter); - _Py_LeaveRecursiveCall(); Py_DECREF(as_parameter); Py_XDECREF(exc); + _Py_LeaveRecursiveCall(); return value; } if (exc) { From d05a4e6a0d366b854a3103cae0c941811fd48c4c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 13 Dec 2024 14:23:20 +0100 Subject: [PATCH 197/427] gh-127906: Test the limited C API in test_cppext (#127916) --- Lib/test/test_cppext/__init__.py | 13 ++++++++++--- Lib/test/test_cppext/extension.cpp | 9 +++++++++ Lib/test/test_cppext/setup.py | 6 ++++++ .../2024-12-13-13-41-34.gh-issue-127906.NuRHlB.rst | 1 + 4 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-12-13-13-41-34.gh-issue-127906.NuRHlB.rst diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index efd79448c66104..d5195227308fec 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -41,12 +41,17 @@ def test_build_cpp11(self): def test_build_cpp14(self): self.check_build('_testcpp14ext', std='c++14') - def check_build(self, extension_name, std=None): + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited(self): + self.check_build('_testcppext_limited', limited=True) + + def check_build(self, extension_name, std=None, limited=False): venv_dir = 'env' with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: - self._check_build(extension_name, python_exe, std=std) + self._check_build(extension_name, python_exe, + std=std, limited=limited) - def _check_build(self, extension_name, python_exe, std): + def _check_build(self, extension_name, python_exe, std, limited): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) @@ -56,6 +61,8 @@ def run_cmd(operation, cmd): env = os.environ.copy() if std: env['CPYTHON_TEST_CPP_STD'] = std + if limited: + env['CPYTHON_TEST_LIMITED'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name if support.verbose: print('Run:', ' '.join(map(shlex.quote, cmd))) diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index ab485b629b7788..500d5918145c00 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -62,6 +62,7 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_ssize_t refcnt = Py_REFCNT(obj); assert(refcnt >= 1); +#ifndef Py_LIMITED_API // gh-92138: For backward compatibility, functions of Python C API accepts // "const PyObject*". Check that using it does not emit C++ compiler // warnings. @@ -74,6 +75,7 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(PyTuple_GET_SIZE(const_obj) == 2); PyObject *one = PyTuple_GET_ITEM(const_obj, 0); assert(PyLong_AsLong(one) == 1); +#endif // gh-92898: StrongRef doesn't inherit from PyObject but has an operator to // cast to PyObject*. @@ -106,6 +108,12 @@ test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } assert(PyUnicode_Check(str)); + + assert(PyUnicode_GetLength(str) == 3); + assert(PyUnicode_ReadChar(str, 0) == 'a'); + assert(PyUnicode_ReadChar(str, 1) == 'b'); + +#ifndef Py_LIMITED_API assert(PyUnicode_GET_LENGTH(str) == 3); // gh-92800: test PyUnicode_READ() @@ -121,6 +129,7 @@ test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(PyUnicode_READ(ukind, const_data, 2) == 'c'); assert(PyUnicode_READ_CHAR(str, 1) == 'b'); +#endif Py_DECREF(str); Py_RETURN_NONE; diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index d97b238b8d1477..019ff18446a2eb 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -33,6 +33,7 @@ def main(): cppflags = list(CPPFLAGS) std = os.environ.get("CPYTHON_TEST_CPP_STD", "") module_name = os.environ["CPYTHON_TEST_EXT_NAME"] + limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) cppflags = list(CPPFLAGS) cppflags.append(f'-DMODULE_NAME={module_name}') @@ -59,6 +60,11 @@ def main(): # CC env var overrides sysconfig CC variable in setuptools os.environ['CC'] = cmd + # Define Py_LIMITED_API macro + if limited: + version = sys.hexversion + cppflags.append(f'-DPy_LIMITED_API={version:#x}') + # On Windows, add PCbuild\amd64\ to include and library directories include_dirs = [] library_dirs = [] diff --git a/Misc/NEWS.d/next/Tests/2024-12-13-13-41-34.gh-issue-127906.NuRHlB.rst b/Misc/NEWS.d/next/Tests/2024-12-13-13-41-34.gh-issue-127906.NuRHlB.rst new file mode 100644 index 00000000000000..6f577e741dff7f --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-12-13-13-41-34.gh-issue-127906.NuRHlB.rst @@ -0,0 +1 @@ +Test the limited C API in test_cppext. Patch by Victor Stinner. From 6446408d426814bf2bc9d3911a91741f04d4bc4e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 13 Dec 2024 14:24:48 +0100 Subject: [PATCH 198/427] gh-102471, PEP 757: Add PyLong import and export API (#121339) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sergey B Kirpichev Co-authored-by: Steve Dower Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/c-api/long.rst | 174 ++++++++++++++++++ Doc/data/refcounts.dat | 7 + Doc/whatsnew/3.14.rst | 11 ++ Include/cpython/longintrepr.h | 38 ++++ Lib/test/test_capi/test_long.py | 91 +++++++++ ...-07-03-17-26-53.gh-issue-102471.XpmKYk.rst | 10 + Modules/_testcapi/long.c | 124 +++++++++++++ Objects/longobject.c | 120 ++++++++++++ Tools/c-analyzer/cpython/ignored.tsv | 1 + 9 files changed, 576 insertions(+) create mode 100644 Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index cb12d43d92026f..f48cd07a979f56 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -653,3 +653,177 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.12 + +Export API +^^^^^^^^^^ + +.. versionadded:: next + +.. c:struct:: PyLongLayout + + Layout of an array of "digits" ("limbs" in the GMP terminology), used to + represent absolute value for arbitrary precision integers. + + Use :c:func:`PyLong_GetNativeLayout` to get the native layout of Python + :class:`int` objects, used internally for integers with "big enough" + absolute value. + + See also :data:`sys.int_info` which exposes similar information in Python. + + .. c:member:: uint8_t bits_per_digit + + Bits per digit. For example, a 15 bit digit means that bits 0-14 contain + meaningful information. + + .. c:member:: uint8_t digit_size + + Digit size in bytes. For example, a 15 bit digit will require at least 2 + bytes. + + .. c:member:: int8_t digits_order + + Digits order: + + - ``1`` for most significant digit first + - ``-1`` for least significant digit first + + .. c:member:: int8_t digit_endianness + + Digit endianness: + + - ``1`` for most significant byte first (big endian) + - ``-1`` for least significant byte first (little endian) + + +.. c:function:: const PyLongLayout* PyLong_GetNativeLayout(void) + + Get the native layout of Python :class:`int` objects. + + See the :c:struct:`PyLongLayout` structure. + + The function must not be called before Python initialization nor after + Python finalization. The returned layout is valid until Python is + finalized. The layout is the same for all Python sub-interpreters + in a process, and so it can be cached. + + +.. c:struct:: PyLongExport + + Export of a Python :class:`int` object. + + There are two cases: + + * If :c:member:`digits` is ``NULL``, only use the :c:member:`value` member. + * If :c:member:`digits` is not ``NULL``, use :c:member:`negative`, + :c:member:`ndigits` and :c:member:`digits` members. + + .. c:member:: int64_t value + + The native integer value of the exported :class:`int` object. + Only valid if :c:member:`digits` is ``NULL``. + + .. c:member:: uint8_t negative + + ``1`` if the number is negative, ``0`` otherwise. + Only valid if :c:member:`digits` is not ``NULL``. + + .. c:member:: Py_ssize_t ndigits + + Number of digits in :c:member:`digits` array. + Only valid if :c:member:`digits` is not ``NULL``. + + .. c:member:: const void *digits + + Read-only array of unsigned digits. Can be ``NULL``. + + +.. c:function:: int PyLong_Export(PyObject *obj, PyLongExport *export_long) + + Export a Python :class:`int` object. + + *export_long* must point to a :c:struct:`PyLongExport` structure allocated + by the caller. It must not be ``NULL``. + + On success, fill in *\*export_long* and return ``0``. + On error, set an exception and return ``-1``. + + :c:func:`PyLong_FreeExport` must be called when the export is no longer + needed. + + .. impl-detail:: + This function always succeeds if *obj* is a Python :class:`int` object + or a subclass. + + +.. c:function:: void PyLong_FreeExport(PyLongExport *export_long) + + Release the export *export_long* created by :c:func:`PyLong_Export`. + + .. impl-detail:: + Calling :c:func:`PyLong_FreeExport` is optional if *export_long->digits* + is ``NULL``. + + +PyLongWriter API +^^^^^^^^^^^^^^^^ + +The :c:type:`PyLongWriter` API can be used to import an integer. + +.. versionadded:: next + +.. c:struct:: PyLongWriter + + A Python :class:`int` writer instance. + + The instance must be destroyed by :c:func:`PyLongWriter_Finish` or + :c:func:`PyLongWriter_Discard`. + + +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) + + Create a :c:type:`PyLongWriter`. + + On success, allocate *\*digits* and return a writer. + On error, set an exception and return ``NULL``. + + *negative* is ``1`` if the number is negative, or ``0`` otherwise. + + *ndigits* is the number of digits in the *digits* array. It must be + greater than 0. + + *digits* must not be NULL. + + After a successful call to this function, the caller should fill in the + array of digits *digits* and then call :c:func:`PyLongWriter_Finish` to get + a Python :class:`int`. + The layout of *digits* is described by :c:func:`PyLong_GetNativeLayout`. + + Digits must be in the range [``0``; ``(1 << bits_per_digit) - 1``] + (where the :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits + per digit). + Any unused most significant digits must be set to ``0``. + + Alternately, call :c:func:`PyLongWriter_Discard` to destroy the writer + instance without creating an :class:`~int` object. + + +.. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) + + Finish a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + + On success, return a Python :class:`int` object. + On error, set an exception and return ``NULL``. + + The function takes care of normalizing the digits and converts the object + to a compact integer if needed. + + The writer instance and the *digits* array are invalid after the call. + + +.. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) + + Discard a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + + *writer* must not be ``NULL``. + + The writer instance and the *digits* array are invalid after the call. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index a043af48ba7a05..e78754e24e23d8 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1299,6 +1299,13 @@ PyLong_GetSign:int::: PyLong_GetSign:PyObject*:v:0: PyLong_GetSign:int*:sign:: +PyLong_Export:int::: +PyLong_Export:PyObject*:obj:0: +PyLong_Export:PyLongExport*:export_long:: + +PyLongWriter_Finish:PyObject*::+1: +PyLongWriter_Finish:PyLongWriter*:writer:: + PyMapping_Check:int::: PyMapping_Check:PyObject*:o:0: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b71d31f9742fe0..5ce398ab93d6b4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1018,6 +1018,17 @@ New features (Contributed by Victor Stinner in :gh:`107954`.) +* Add a new import and export API for Python :class:`int` objects (:pep:`757`): + + * :c:func:`PyLong_GetNativeLayout`; + * :c:func:`PyLong_Export`; + * :c:func:`PyLong_FreeExport`; + * :c:func:`PyLongWriter_Create`; + * :c:func:`PyLongWriter_Finish`; + * :c:func:`PyLongWriter_Discard`. + + (Contributed by Victor Stinner in :gh:`102471`.) + * Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for easier superclass identification, which attempts to resolve the `type checking issue `__ mentioned in :pep:`630` diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index c60ccc463653f9..357477b60d9a5a 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -139,6 +139,44 @@ _PyLong_CompactValue(const PyLongObject *op) #define PyUnstable_Long_CompactValue _PyLong_CompactValue +/* --- Import/Export API -------------------------------------------------- */ + +typedef struct PyLongLayout { + uint8_t bits_per_digit; + uint8_t digit_size; + int8_t digits_order; + int8_t digit_endianness; +} PyLongLayout; + +PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); + +typedef struct PyLongExport { + int64_t value; + uint8_t negative; + Py_ssize_t ndigits; + const void *digits; + // Member used internally, must not be used for other purpose. + Py_uintptr_t _reserved; +} PyLongExport; + +PyAPI_FUNC(int) PyLong_Export( + PyObject *obj, + PyLongExport *export_long); +PyAPI_FUNC(void) PyLong_FreeExport( + PyLongExport *export_long); + + +/* --- PyLongWriter API --------------------------------------------------- */ + +typedef struct PyLongWriter PyLongWriter; + +PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( + int negative, + Py_ssize_t ndigits, + void **digits); +PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); +PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index a77094588a0edf..d45ac75c822ea9 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -10,6 +10,7 @@ NULL = None + class IntSubclass(int): pass @@ -714,5 +715,95 @@ def test_long_asuint64(self): self.check_long_asint(as_uint64, 0, UINT64_MAX, negative_value_error=ValueError) + def test_long_layout(self): + # Test PyLong_GetNativeLayout() + int_info = sys.int_info + layout = _testcapi.get_pylong_layout() + expected = { + 'bits_per_digit': int_info.bits_per_digit, + 'digit_size': int_info.sizeof_digit, + 'digits_order': -1, + 'digit_endianness': -1 if sys.byteorder == 'little' else 1, + } + self.assertEqual(layout, expected) + + def test_long_export(self): + # Test PyLong_Export() + layout = _testcapi.get_pylong_layout() + base = 2 ** layout['bits_per_digit'] + + pylong_export = _testcapi.pylong_export + + # value fits into int64_t + self.assertEqual(pylong_export(0), 0) + self.assertEqual(pylong_export(123), 123) + self.assertEqual(pylong_export(-123), -123) + self.assertEqual(pylong_export(IntSubclass(123)), 123) + + # use an array, doesn't fit into int64_t + self.assertEqual(pylong_export(base**10 * 2 + 1), + (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + self.assertEqual(pylong_export(-(base**10 * 2 + 1)), + (1, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + self.assertEqual(pylong_export(IntSubclass(base**10 * 2 + 1)), + (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + + self.assertRaises(TypeError, pylong_export, 1.0) + self.assertRaises(TypeError, pylong_export, 0+1j) + self.assertRaises(TypeError, pylong_export, "abc") + + def test_longwriter_create(self): + # Test PyLongWriter_Create() + layout = _testcapi.get_pylong_layout() + base = 2 ** layout['bits_per_digit'] + + pylongwriter_create = _testcapi.pylongwriter_create + self.assertRaises(ValueError, pylongwriter_create, 0, []) + self.assertRaises(ValueError, pylongwriter_create, -123, []) + self.assertEqual(pylongwriter_create(0, [0]), 0) + self.assertEqual(pylongwriter_create(0, [123]), 123) + self.assertEqual(pylongwriter_create(1, [123]), -123) + self.assertEqual(pylongwriter_create(1, [1, 2]), + -(base * 2 + 1)) + self.assertEqual(pylongwriter_create(0, [1, 2, 3]), + base**2 * 3 + base * 2 + 1) + max_digit = base - 1 + self.assertEqual(pylongwriter_create(0, [max_digit, max_digit, max_digit]), + base**2 * max_digit + base * max_digit + max_digit) + + # normalize + self.assertEqual(pylongwriter_create(0, [123, 0, 0]), 123) + + # test singletons + normalize + for num in (-2, 0, 1, 5, 42, 100): + self.assertIs(pylongwriter_create(bool(num < 0), [abs(num), 0]), + num) + + def to_digits(num): + digits = [] + while True: + num, digit = divmod(num, base) + digits.append(digit) + if not num: + break + return digits + + # round trip: Python int -> export -> Python int + pylong_export = _testcapi.pylong_export + numbers = [*range(0, 10), 12345, 0xdeadbeef, 2**100, 2**100-1] + numbers.extend(-num for num in list(numbers)) + for num in numbers: + with self.subTest(num=num): + data = pylong_export(num) + if isinstance(data, tuple): + negative, digits = data + else: + value = data + negative = int(value < 0) + digits = to_digits(abs(value)) + self.assertEqual(pylongwriter_create(negative, digits), num, + (negative, digits)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst b/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst new file mode 100644 index 00000000000000..c18c159ac87d08 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst @@ -0,0 +1,10 @@ +Add a new import and export API for Python :class:`int` objects (:pep:`757`): + +* :c:func:`PyLong_GetNativeLayout`; +* :c:func:`PyLong_Export`; +* :c:func:`PyLong_FreeExport`; +* :c:func:`PyLongWriter_Create`; +* :c:func:`PyLongWriter_Finish`; +* :c:func:`PyLongWriter_Discard`. + +Patch by Victor Stinner. diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index ebea09080ef11c..42243023a45768 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -141,6 +141,127 @@ pylong_aspid(PyObject *module, PyObject *arg) } +static PyObject * +layout_to_dict(const PyLongLayout *layout) +{ + return Py_BuildValue("{sisisisi}", + "bits_per_digit", (int)layout->bits_per_digit, + "digit_size", (int)layout->digit_size, + "digits_order", (int)layout->digits_order, + "digit_endianness", (int)layout->digit_endianness); +} + + +static PyObject * +pylong_export(PyObject *module, PyObject *obj) +{ + PyLongExport export_long; + if (PyLong_Export(obj, &export_long) < 0) { + return NULL; + } + + if (export_long.digits == NULL) { + assert(export_long.negative == 0); + assert(export_long.ndigits == 0); + assert(export_long.digits == NULL); + PyObject *res = PyLong_FromInt64(export_long.value); + PyLong_FreeExport(&export_long); + return res; + } + + assert(PyLong_GetNativeLayout()->digit_size == sizeof(digit)); + const digit *export_long_digits = export_long.digits; + + PyObject *digits = PyList_New(0); + if (digits == NULL) { + goto error; + } + for (Py_ssize_t i = 0; i < export_long.ndigits; i++) { + PyObject *item = PyLong_FromUnsignedLong(export_long_digits[i]); + if (item == NULL) { + goto error; + } + + if (PyList_Append(digits, item) < 0) { + Py_DECREF(item); + goto error; + } + Py_DECREF(item); + } + + assert(export_long.value == 0); + PyObject *res = Py_BuildValue("(iN)", export_long.negative, digits); + + PyLong_FreeExport(&export_long); + assert(export_long._reserved == 0); + + return res; + +error: + Py_XDECREF(digits); + PyLong_FreeExport(&export_long); + return NULL; +} + + +static PyObject * +pylongwriter_create(PyObject *module, PyObject *args) +{ + int negative; + PyObject *list; + // TODO(vstinner): write test for negative ndigits and digits==NULL + if (!PyArg_ParseTuple(args, "iO!", &negative, &PyList_Type, &list)) { + return NULL; + } + Py_ssize_t ndigits = PyList_GET_SIZE(list); + + digit *digits = PyMem_Malloc((size_t)ndigits * sizeof(digit)); + if (digits == NULL) { + return PyErr_NoMemory(); + } + + for (Py_ssize_t i = 0; i < ndigits; i++) { + PyObject *item = PyList_GET_ITEM(list, i); + + long num = PyLong_AsLong(item); + if (num == -1 && PyErr_Occurred()) { + goto error; + } + + if (num < 0 || num >= PyLong_BASE) { + PyErr_SetString(PyExc_ValueError, "digit doesn't fit into digit"); + goto error; + } + digits[i] = (digit)num; + } + + void *writer_digits; + PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, + &writer_digits); + if (writer == NULL) { + goto error; + } + assert(PyLong_GetNativeLayout()->digit_size == sizeof(digit)); + memcpy(writer_digits, digits, (size_t)ndigits * sizeof(digit)); + PyObject *res = PyLongWriter_Finish(writer); + PyMem_Free(digits); + + return res; + +error: + PyMem_Free(digits); + return NULL; +} + + +static PyObject * +get_pylong_layout(PyObject *module, PyObject *Py_UNUSED(args)) +{ + const PyLongLayout *layout = PyLong_GetNativeLayout(); + return layout_to_dict(layout); +} + + static PyMethodDef test_methods[] = { _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF {"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS}, @@ -148,6 +269,9 @@ static PyMethodDef test_methods[] = { {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, {"pylong_getsign", pylong_getsign, METH_O}, {"pylong_aspid", pylong_aspid, METH_O}, + {"pylong_export", pylong_export, METH_O}, + {"pylongwriter_create", pylongwriter_create, METH_VARARGS}, + {"get_pylong_layout", get_pylong_layout, METH_NOARGS}, {"pylong_ispositive", pylong_ispositive, METH_O}, {"pylong_isnegative", pylong_isnegative, METH_O}, {"pylong_iszero", pylong_iszero, METH_O}, diff --git a/Objects/longobject.c b/Objects/longobject.c index 96d59f542a7c3c..bd7ff68d0899c6 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6750,6 +6750,7 @@ PyUnstable_Long_CompactValue(const PyLongObject* op) { return _PyLong_CompactValue((PyLongObject*)op); } + PyObject* PyLong_FromInt32(int32_t value) { return PyLong_FromNativeBytes(&value, sizeof(value), -1); } @@ -6815,3 +6816,122 @@ int PyLong_AsUInt64(PyObject *obj, uint64_t *value) { LONG_TO_UINT(obj, value, "C uint64_t"); } + + +static const PyLongLayout PyLong_LAYOUT = { + .bits_per_digit = PyLong_SHIFT, + .digits_order = -1, // least significant first + .digit_endianness = PY_LITTLE_ENDIAN ? -1 : 1, + .digit_size = sizeof(digit), +}; + + +const PyLongLayout* +PyLong_GetNativeLayout(void) +{ + return &PyLong_LAYOUT; +} + + +int +PyLong_Export(PyObject *obj, PyLongExport *export_long) +{ + if (!PyLong_Check(obj)) { + memset(export_long, 0, sizeof(*export_long)); + PyErr_Format(PyExc_TypeError, "expect int, got %T", obj); + return -1; + } + + // Fast-path: try to convert to a int64_t + int overflow; +#if SIZEOF_LONG == 8 + long value = PyLong_AsLongAndOverflow(obj, &overflow); +#else + // Windows has 32-bit long, so use 64-bit long long instead + long long value = PyLong_AsLongLongAndOverflow(obj, &overflow); +#endif + Py_BUILD_ASSERT(sizeof(value) == sizeof(int64_t)); + // the function cannot fail since obj is a PyLongObject + assert(!(value == -1 && PyErr_Occurred())); + + if (!overflow) { + export_long->value = value; + export_long->negative = 0; + export_long->ndigits = 0; + export_long->digits = NULL; + export_long->_reserved = 0; + } + else { + PyLongObject *self = (PyLongObject*)obj; + export_long->value = 0; + export_long->negative = _PyLong_IsNegative(self); + export_long->ndigits = _PyLong_DigitCount(self); + if (export_long->ndigits == 0) { + export_long->ndigits = 1; + } + export_long->digits = self->long_value.ob_digit; + export_long->_reserved = (Py_uintptr_t)Py_NewRef(obj); + } + return 0; +} + + +void +PyLong_FreeExport(PyLongExport *export_long) +{ + PyObject *obj = (PyObject*)export_long->_reserved; + if (obj) { + export_long->_reserved = 0; + Py_DECREF(obj); + } +} + + +/* --- PyLongWriter API --------------------------------------------------- */ + +PyLongWriter* +PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) +{ + if (ndigits <= 0) { + PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); + goto error; + } + assert(digits != NULL); + + PyLongObject *obj = _PyLong_New(ndigits); + if (obj == NULL) { + goto error; + } + if (negative) { + _PyLong_FlipSign(obj); + } + + *digits = obj->long_value.ob_digit; + return (PyLongWriter*)obj; + +error: + *digits = NULL; + return NULL; +} + + +void +PyLongWriter_Discard(PyLongWriter *writer) +{ + PyLongObject *obj = (PyLongObject *)writer; + assert(Py_REFCNT(obj) == 1); + Py_DECREF(obj); +} + + +PyObject* +PyLongWriter_Finish(PyLongWriter *writer) +{ + PyLongObject *obj = (PyLongObject *)writer; + assert(Py_REFCNT(obj) == 1); + + // Normalize and get singleton if possible + obj = maybe_small_long(long_normalize(obj)); + + return (PyObject*)obj; +} diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 686f3935d91bda..c8c30a7985aa2e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -319,6 +319,7 @@ Objects/exceptions.c - static_exceptions - Objects/genobject.c - ASYNC_GEN_IGNORED_EXIT_MSG - Objects/genobject.c - NON_INIT_CORO_MSG - Objects/longobject.c - _PyLong_DigitValue - +Objects/longobject.c - PyLong_LAYOUT - Objects/object.c - _Py_SwappedOp - Objects/object.c - _Py_abstract_hack - Objects/object.c - last_final_reftotal - From 8bc18182a7c28f86265c9d82bd0338137480921c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:16:22 +0100 Subject: [PATCH 199/427] gh-127691: add type checks when using `PyUnicodeError` objects (GH-127694) --- Doc/whatsnew/3.14.rst | 6 + ...-12-06-16-53-34.gh-issue-127691.k_Jitp.rst | 3 + Objects/exceptions.c | 216 ++++++++++++------ 3 files changed, 157 insertions(+), 68 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-12-06-16-53-34.gh-issue-127691.k_Jitp.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 5ce398ab93d6b4..095949242c09d9 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1045,6 +1045,12 @@ New features * Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling deferred reference counting, as outlined in :pep:`703`. +* The :ref:`Unicode Exception Objects ` C API + now raises a :exc:`TypeError` if its exception argument is not + a :exc:`UnicodeError` object. + (Contributed by Bénédikt Tran in :gh:`127691`.) + + Porting to Python 3.14 ---------------------- diff --git a/Misc/NEWS.d/next/C_API/2024-12-06-16-53-34.gh-issue-127691.k_Jitp.rst b/Misc/NEWS.d/next/C_API/2024-12-06-16-53-34.gh-issue-127691.k_Jitp.rst new file mode 100644 index 00000000000000..c942ff3d9eda53 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-06-16-53-34.gh-issue-127691.k_Jitp.rst @@ -0,0 +1,3 @@ +The :ref:`Unicode Exception Objects ` C API +now raises a :exc:`TypeError` if its exception argument is not +a :exc:`UnicodeError` object. Patch by Bénédikt Tran. diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 287cbc25305964..6880c24196cbb8 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2668,7 +2668,7 @@ SimpleExtendsException(PyExc_ValueError, UnicodeError, "Unicode related error."); static PyObject * -get_string(PyObject *attr, const char *name) +get_bytes(PyObject *attr, const char *name) { if (!attr) { PyErr_Format(PyExc_TypeError, "%.200s attribute not set", name); @@ -2748,40 +2748,74 @@ unicode_error_adjust_end(Py_ssize_t end, Py_ssize_t objlen) return end; } +#define _PyUnicodeError_CAST(PTR) ((PyUnicodeErrorObject *)(PTR)) +#define PyUnicodeError_Check(PTR) \ + PyObject_TypeCheck((PTR), (PyTypeObject *)PyExc_UnicodeError) +#define PyUnicodeError_CAST(PTR) \ + (assert(PyUnicodeError_Check(PTR)), _PyUnicodeError_CAST(PTR)) + + +static inline int +check_unicode_error_type(PyObject *self, const char *expect_type) +{ + if (!PyUnicodeError_Check(self)) { + PyErr_Format(PyExc_TypeError, + "expecting a %s object, got %T", expect_type, self); + return -1; + } + return 0; +} + + +static inline PyUnicodeErrorObject * +as_unicode_error(PyObject *self, const char *expect_type) +{ + int rc = check_unicode_error_type(self, expect_type); + return rc < 0 ? NULL : _PyUnicodeError_CAST(self); +} + PyObject * -PyUnicodeEncodeError_GetEncoding(PyObject *exc) +PyUnicodeEncodeError_GetEncoding(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->encoding, "encoding"); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); + return exc == NULL ? NULL : get_unicode(exc->encoding, "encoding"); } PyObject * -PyUnicodeDecodeError_GetEncoding(PyObject *exc) +PyUnicodeDecodeError_GetEncoding(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->encoding, "encoding"); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); + return exc == NULL ? NULL : get_unicode(exc->encoding, "encoding"); } PyObject * -PyUnicodeEncodeError_GetObject(PyObject *exc) +PyUnicodeEncodeError_GetObject(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->object, "object"); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); + return exc == NULL ? NULL : get_unicode(exc->object, "object"); } PyObject * -PyUnicodeDecodeError_GetObject(PyObject *exc) +PyUnicodeDecodeError_GetObject(PyObject *self) { - return get_string(((PyUnicodeErrorObject *)exc)->object, "object"); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); + return exc == NULL ? NULL : get_bytes(exc->object, "object"); } PyObject * -PyUnicodeTranslateError_GetObject(PyObject *exc) +PyUnicodeTranslateError_GetObject(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->object, "object"); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeTranslateError"); + return exc == NULL ? NULL : get_unicode(exc->object, "object"); } int PyUnicodeEncodeError_GetStart(PyObject *self, Py_ssize_t *start) { - PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); + if (exc == NULL) { + return -1; + } PyObject *obj = get_unicode(exc->object, "object"); if (obj == NULL) { return -1; @@ -2796,8 +2830,11 @@ PyUnicodeEncodeError_GetStart(PyObject *self, Py_ssize_t *start) int PyUnicodeDecodeError_GetStart(PyObject *self, Py_ssize_t *start) { - PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; - PyObject *obj = get_string(exc->object, "object"); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); + if (exc == NULL) { + return -1; + } + PyObject *obj = get_bytes(exc->object, "object"); if (obj == NULL) { return -1; } @@ -2809,45 +2846,63 @@ PyUnicodeDecodeError_GetStart(PyObject *self, Py_ssize_t *start) int -PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) +PyUnicodeTranslateError_GetStart(PyObject *self, Py_ssize_t *start) { - return PyUnicodeEncodeError_GetStart(exc, start); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeTranslateError"); + if (exc == NULL) { + return -1; + } + PyObject *obj = get_unicode(exc->object, "object"); + if (obj == NULL) { + return -1; + } + Py_ssize_t size = PyUnicode_GET_LENGTH(obj); + Py_DECREF(obj); + *start = unicode_error_adjust_start(exc->start, size); + return 0; } static inline int unicode_error_set_start_impl(PyObject *self, Py_ssize_t start) { - ((PyUnicodeErrorObject *)self)->start = start; + PyUnicodeErrorObject *exc = _PyUnicodeError_CAST(self); + exc->start = start; return 0; } int -PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) +PyUnicodeEncodeError_SetStart(PyObject *self, Py_ssize_t start) { - return unicode_error_set_start_impl(exc, start); + int rc = check_unicode_error_type(self, "UnicodeEncodeError"); + return rc < 0 ? -1 : unicode_error_set_start_impl(self, start); } int -PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) +PyUnicodeDecodeError_SetStart(PyObject *self, Py_ssize_t start) { - return unicode_error_set_start_impl(exc, start); + int rc = check_unicode_error_type(self, "UnicodeDecodeError"); + return rc < 0 ? -1 : unicode_error_set_start_impl(self, start); } int -PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) +PyUnicodeTranslateError_SetStart(PyObject *self, Py_ssize_t start) { - return unicode_error_set_start_impl(exc, start); + int rc = check_unicode_error_type(self, "UnicodeTranslateError"); + return rc < 0 ? -1 : unicode_error_set_start_impl(self, start); } int PyUnicodeEncodeError_GetEnd(PyObject *self, Py_ssize_t *end) { - PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); + if (exc == NULL) { + return -1; + } PyObject *obj = get_unicode(exc->object, "object"); if (obj == NULL) { return -1; @@ -2862,8 +2917,11 @@ PyUnicodeEncodeError_GetEnd(PyObject *self, Py_ssize_t *end) int PyUnicodeDecodeError_GetEnd(PyObject *self, Py_ssize_t *end) { - PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; - PyObject *obj = get_string(exc->object, "object"); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); + if (exc == NULL) { + return -1; + } + PyObject *obj = get_bytes(exc->object, "object"); if (obj == NULL) { return -1; } @@ -2875,108 +2933,130 @@ PyUnicodeDecodeError_GetEnd(PyObject *self, Py_ssize_t *end) int -PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) +PyUnicodeTranslateError_GetEnd(PyObject *self, Py_ssize_t *end) { - return PyUnicodeEncodeError_GetEnd(exc, end); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeTranslateError"); + if (exc == NULL) { + return -1; + } + PyObject *obj = get_unicode(exc->object, "object"); + if (obj == NULL) { + return -1; + } + Py_ssize_t size = PyUnicode_GET_LENGTH(obj); + Py_DECREF(obj); + *end = unicode_error_adjust_end(exc->end, size); + return 0; } static inline int -unicode_error_set_end_impl(PyObject *exc, Py_ssize_t end) +unicode_error_set_end_impl(PyObject *self, Py_ssize_t end) { - ((PyUnicodeErrorObject *)exc)->end = end; + PyUnicodeErrorObject *exc = _PyUnicodeError_CAST(self); + exc->end = end; return 0; } int -PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) +PyUnicodeEncodeError_SetEnd(PyObject *self, Py_ssize_t end) { - return unicode_error_set_end_impl(exc, end); + int rc = check_unicode_error_type(self, "UnicodeEncodeError"); + return rc < 0 ? -1 : unicode_error_set_end_impl(self, end); } int -PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) +PyUnicodeDecodeError_SetEnd(PyObject *self, Py_ssize_t end) { - return unicode_error_set_end_impl(exc, end); + int rc = check_unicode_error_type(self, "UnicodeDecodeError"); + return rc < 0 ? -1 : unicode_error_set_end_impl(self, end); } int -PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) +PyUnicodeTranslateError_SetEnd(PyObject *self, Py_ssize_t end) { - return unicode_error_set_end_impl(exc, end); + int rc = check_unicode_error_type(self, "UnicodeTranslateError"); + return rc < 0 ? -1 : unicode_error_set_end_impl(self, end); } + PyObject * -PyUnicodeEncodeError_GetReason(PyObject *exc) +PyUnicodeEncodeError_GetReason(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->reason, "reason"); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); + return exc == NULL ? NULL : get_unicode(exc->reason, "reason"); } PyObject * -PyUnicodeDecodeError_GetReason(PyObject *exc) +PyUnicodeDecodeError_GetReason(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->reason, "reason"); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); + return exc == NULL ? NULL : get_unicode(exc->reason, "reason"); } PyObject * -PyUnicodeTranslateError_GetReason(PyObject *exc) +PyUnicodeTranslateError_GetReason(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->reason, "reason"); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeTranslateError"); + return exc == NULL ? NULL : get_unicode(exc->reason, "reason"); } int -PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) +PyUnicodeEncodeError_SetReason(PyObject *self, const char *reason) { - return set_unicodefromstring(&((PyUnicodeErrorObject *)exc)->reason, - reason); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); + return exc == NULL ? -1 : set_unicodefromstring(&exc->reason, reason); } int -PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) +PyUnicodeDecodeError_SetReason(PyObject *self, const char *reason) { - return set_unicodefromstring(&((PyUnicodeErrorObject *)exc)->reason, - reason); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); + return exc == NULL ? -1 : set_unicodefromstring(&exc->reason, reason); } int -PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) +PyUnicodeTranslateError_SetReason(PyObject *self, const char *reason) { - return set_unicodefromstring(&((PyUnicodeErrorObject *)exc)->reason, - reason); + PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeTranslateError"); + return exc == NULL ? -1 : set_unicodefromstring(&exc->reason, reason); } static int -UnicodeError_clear(PyUnicodeErrorObject *self) +UnicodeError_clear(PyObject *self) { - Py_CLEAR(self->encoding); - Py_CLEAR(self->object); - Py_CLEAR(self->reason); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + Py_CLEAR(exc->encoding); + Py_CLEAR(exc->object); + Py_CLEAR(exc->reason); return BaseException_clear((PyBaseExceptionObject *)self); } static void -UnicodeError_dealloc(PyUnicodeErrorObject *self) +UnicodeError_dealloc(PyObject *self) { + PyTypeObject *type = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - UnicodeError_clear(self); - Py_TYPE(self)->tp_free((PyObject *)self); + (void)UnicodeError_clear(self); + type->tp_free(self); } static int -UnicodeError_traverse(PyUnicodeErrorObject *self, visitproc visit, void *arg) +UnicodeError_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(self->encoding); - Py_VISIT(self->object); - Py_VISIT(self->reason); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + Py_VISIT(exc->encoding); + Py_VISIT(exc->object); + Py_VISIT(exc->reason); return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg); } @@ -3015,7 +3095,7 @@ UnicodeEncodeError_init(PyObject *self, PyObject *args, PyObject *kwds) return -1; } - PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); Py_XSETREF(exc->encoding, Py_NewRef(encoding)); Py_XSETREF(exc->object, Py_NewRef(object)); exc->start = start; @@ -3027,7 +3107,7 @@ UnicodeEncodeError_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * UnicodeEncodeError_str(PyObject *self) { - PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); PyObject *result = NULL; PyObject *reason_str = NULL; PyObject *encoding_str = NULL; @@ -3135,7 +3215,7 @@ UnicodeDecodeError_init(PyObject *self, PyObject *args, PyObject *kwds) } } - PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); Py_XSETREF(exc->encoding, Py_NewRef(encoding)); Py_XSETREF(exc->object, object /* already a strong reference */); exc->start = start; @@ -3147,7 +3227,7 @@ UnicodeDecodeError_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * UnicodeDecodeError_str(PyObject *self) { - PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); PyObject *result = NULL; PyObject *reason_str = NULL; PyObject *encoding_str = NULL; @@ -3236,7 +3316,7 @@ UnicodeTranslateError_init(PyObject *self, PyObject *args, PyObject *kwds) return -1; } - PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); Py_XSETREF(exc->object, Py_NewRef(object)); exc->start = start; exc->end = end; @@ -3248,7 +3328,7 @@ UnicodeTranslateError_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * UnicodeTranslateError_str(PyObject *self) { - PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); PyObject *result = NULL; PyObject *reason_str = NULL; From 5dd775bed086909722ec7014a7c4f77a35f74a80 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 14 Dec 2024 01:21:46 +0900 Subject: [PATCH 200/427] gh-126024: unicodeobject: optimize find_first_nonascii (GH-127790) Remove 1 branch. --- Objects/unicodeobject.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 33c4747bbef488..b7aeb06d32bcec 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5077,21 +5077,24 @@ load_unaligned(const unsigned char *p, size_t size) static Py_ssize_t find_first_nonascii(const unsigned char *start, const unsigned char *end) { + // The search is done in `size_t` chunks. + // The start and end might not be aligned at `size_t` boundaries, + // so they're handled specially. + const unsigned char *p = start; if (end - start >= SIZEOF_SIZE_T) { - const unsigned char *p2 = _Py_ALIGN_UP(p, SIZEOF_SIZE_T); + // Avoid unaligned read. #if PY_LITTLE_ENDIAN && HAVE_CTZ - if (p < p2) { - size_t u; - memcpy(&u, p, sizeof(size_t)); - u &= ASCII_CHAR_MASK; - if (u) { - return (ctz(u) - 7) / 8; - } - p = p2; + size_t u; + memcpy(&u, p, sizeof(size_t)); + u &= ASCII_CHAR_MASK; + if (u) { + return (ctz(u) - 7) / 8; } + p = _Py_ALIGN_DOWN(p + SIZEOF_SIZE_T, SIZEOF_SIZE_T); #else /* PY_LITTLE_ENDIAN && HAVE_CTZ */ + const unsigned char *p2 = _Py_ALIGN_UP(p, SIZEOF_SIZE_T); while (p < p2) { if (*p & 0x80) { return p - start; @@ -5099,6 +5102,7 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) p++; } #endif + const unsigned char *e = end - SIZEOF_SIZE_T; while (p <= e) { size_t u = (*(const size_t *)p) & ASCII_CHAR_MASK; @@ -5115,6 +5119,7 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) } } #if PY_LITTLE_ENDIAN && HAVE_CTZ + assert((end - p) < SIZEOF_SIZE_T); // we can not use *(const size_t*)p to avoid buffer overrun. size_t u = load_unaligned(p, end - p) & ASCII_CHAR_MASK; if (u) { From 292067fbc9db81896c16ff12d51c21d2b0f233e2 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 13 Dec 2024 12:12:49 -0600 Subject: [PATCH 201/427] Minor readability improvements for the itertools recipes (gh-127928) --- Doc/library/itertools.rst | 74 ++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 3b90d7830f3681..d1fb952e36f686 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -30,11 +30,6 @@ For instance, SML provides a tabulation tool: ``tabulate(f)`` which produces a sequence ``f(0), f(1), ...``. The same effect can be achieved in Python by combining :func:`map` and :func:`count` to form ``map(f, count())``. -These tools and their built-in counterparts also work well with the high-speed -functions in the :mod:`operator` module. For example, the multiplication -operator can be mapped across two vectors to form an efficient dot-product: -``sum(starmap(operator.mul, zip(vec1, vec2, strict=True)))``. - **Infinite iterators:** @@ -843,12 +838,11 @@ and :term:`generators ` which incur interpreter overhead. .. testcode:: - import collections - import contextlib - import functools - import math - import operator - import random + from collections import deque + from contextlib import suppress + from functools import reduce + from math import sumprod, isqrt + from operator import itemgetter, getitem, mul, neg def take(n, iterable): "Return first n items of the iterable as a list." @@ -863,11 +857,11 @@ and :term:`generators ` which incur interpreter overhead. "Return function(0), function(1), ..." return map(function, count(start)) - def repeatfunc(func, times=None, *args): - "Repeat calls to func with specified arguments." + def repeatfunc(function, times=None, *args): + "Repeat calls to a function with specified arguments." if times is None: - return starmap(func, repeat(args)) - return starmap(func, repeat(args, times)) + return starmap(function, repeat(args)) + return starmap(function, repeat(args, times)) def flatten(list_of_lists): "Flatten one level of nesting." @@ -885,13 +879,13 @@ and :term:`generators ` which incur interpreter overhead. def tail(n, iterable): "Return an iterator over the last n items." # tail(3, 'ABCDEFG') → E F G - return iter(collections.deque(iterable, maxlen=n)) + return iter(deque(iterable, maxlen=n)) def consume(iterator, n=None): "Advance the iterator n-steps ahead. If n is None, consume entirely." # Use functions that consume iterators at C speed. if n is None: - collections.deque(iterator, maxlen=0) + deque(iterator, maxlen=0) else: next(islice(iterator, n, n), None) @@ -919,8 +913,8 @@ and :term:`generators ` which incur interpreter overhead. # unique_justseen('AAAABBBCCDAABBB') → A B C D A B # unique_justseen('ABBcCAD', str.casefold) → A B c A D if key is None: - return map(operator.itemgetter(0), groupby(iterable)) - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + return map(itemgetter(0), groupby(iterable)) + return map(next, map(itemgetter(1), groupby(iterable, key))) def unique_everseen(iterable, key=None): "Yield unique elements, preserving order. Remember all elements ever seen." @@ -941,13 +935,14 @@ and :term:`generators ` which incur interpreter overhead. def unique(iterable, key=None, reverse=False): "Yield unique elements in sorted order. Supports unhashable inputs." # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] - return unique_justseen(sorted(iterable, key=key, reverse=reverse), key=key) + sequenced = sorted(iterable, key=key, reverse=reverse) + return unique_justseen(sequenced, key=key) def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG iterator = iter(iterable) - window = collections.deque(islice(iterator, n - 1), maxlen=n) + window = deque(islice(iterator, n - 1), maxlen=n) for x in iterator: window.append(x) yield tuple(window) @@ -981,7 +976,7 @@ and :term:`generators ` which incur interpreter overhead. "Return all contiguous non-empty subslices of a sequence." # subslices('ABCD') → A AB ABC ABCD B BC BCD C CD D slices = starmap(slice, combinations(range(len(seq) + 1), 2)) - return map(operator.getitem, repeat(seq), slices) + return map(getitem, repeat(seq), slices) def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." @@ -995,19 +990,19 @@ and :term:`generators ` which incur interpreter overhead. else: stop = len(iterable) if stop is None else stop i = start - with contextlib.suppress(ValueError): + with suppress(ValueError): while True: yield (i := seq_index(value, i, stop)) i += 1 - def iter_except(func, exception, first=None): + def iter_except(function, exception, first=None): "Convert a call-until-exception interface to an iterator interface." # iter_except(d.popitem, KeyError) → non-blocking dictionary iterator - with contextlib.suppress(exception): + with suppress(exception): if first is not None: yield first() while True: - yield func() + yield function() The following recipes have a more mathematical flavor: @@ -1015,6 +1010,7 @@ The following recipes have a more mathematical flavor: .. testcode:: def powerset(iterable): + "Subsequences of the iterable from shortest to longest." # powerset([1,2,3]) → () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3) s = list(iterable) return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) @@ -1022,12 +1018,12 @@ The following recipes have a more mathematical flavor: def sum_of_squares(iterable): "Add up the squares of the input values." # sum_of_squares([10, 20, 30]) → 1400 - return math.sumprod(*tee(iterable)) + return sumprod(*tee(iterable)) - def reshape(matrix, cols): + def reshape(matrix, columns): "Reshape a 2-D matrix to have a given number of columns." # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2), (3, 4, 5) - return batched(chain.from_iterable(matrix), cols, strict=True) + return batched(chain.from_iterable(matrix), columns, strict=True) def transpose(matrix): "Swap the rows and columns of a 2-D matrix." @@ -1038,7 +1034,7 @@ The following recipes have a more mathematical flavor: "Multiply two matrices." # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80), (41, 60) n = len(m2[0]) - return batched(starmap(math.sumprod, product(m1, transpose(m2))), n) + return batched(starmap(sumprod, product(m1, transpose(m2))), n) def convolve(signal, kernel): """Discrete linear convolution of two iterables. @@ -1059,7 +1055,7 @@ The following recipes have a more mathematical flavor: n = len(kernel) padded_signal = chain(repeat(0, n-1), signal, repeat(0, n-1)) windowed_signal = sliding_window(padded_signal, n) - return map(math.sumprod, repeat(kernel), windowed_signal) + return map(sumprod, repeat(kernel), windowed_signal) def polynomial_from_roots(roots): """Compute a polynomial's coefficients from its roots. @@ -1067,8 +1063,8 @@ The following recipes have a more mathematical flavor: (x - 5) (x + 4) (x - 3) expands to: x³ -4x² -17x + 60 """ # polynomial_from_roots([5, -4, 3]) → [1, -4, -17, 60] - factors = zip(repeat(1), map(operator.neg, roots)) - return list(functools.reduce(convolve, factors, [1])) + factors = zip(repeat(1), map(neg, roots)) + return list(reduce(convolve, factors, [1])) def polynomial_eval(coefficients, x): """Evaluate a polynomial at a specific value. @@ -1081,7 +1077,7 @@ The following recipes have a more mathematical flavor: if not n: return type(x)(0) powers = map(pow, repeat(x), reversed(range(n))) - return math.sumprod(coefficients, powers) + return sumprod(coefficients, powers) def polynomial_derivative(coefficients): """Compute the first derivative of a polynomial. @@ -1092,7 +1088,7 @@ The following recipes have a more mathematical flavor: # polynomial_derivative([1, -4, -17, 60]) → [3, -8, -17] n = len(coefficients) powers = reversed(range(1, n)) - return list(map(operator.mul, coefficients, powers)) + return list(map(mul, coefficients, powers)) def sieve(n): "Primes less than n." @@ -1100,7 +1096,7 @@ The following recipes have a more mathematical flavor: if n > 2: yield 2 data = bytearray((0, 1)) * (n // 2) - for p in iter_index(data, 1, start=3, stop=math.isqrt(n) + 1): + for p in iter_index(data, 1, start=3, stop=isqrt(n) + 1): data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) yield from iter_index(data, 1, start=3) @@ -1109,7 +1105,7 @@ The following recipes have a more mathematical flavor: # factor(99) → 3 3 11 # factor(1_000_000_000_000_007) → 47 59 360620266859 # factor(1_000_000_000_000_403) → 1000000000000403 - for prime in sieve(math.isqrt(n) + 1): + for prime in sieve(isqrt(n) + 1): while not n % prime: yield prime n //= prime @@ -1740,7 +1736,7 @@ The following recipes have a more mathematical flavor: # Old recipes and their tests which are guaranteed to continue to work. - def sumprod(vec1, vec2): + def old_sumprod_recipe(vec1, vec2): "Compute a sum of products." return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) @@ -1823,7 +1819,7 @@ The following recipes have a more mathematical flavor: 32 - >>> sumprod([1,2,3], [4,5,6]) + >>> old_sumprod_recipe([1,2,3], [4,5,6]) 32 From 2de048ce79e621f5ae0574095b9600fe8595f607 Mon Sep 17 00:00:00 2001 From: mpage Date: Fri, 13 Dec 2024 10:17:16 -0800 Subject: [PATCH 202/427] gh-115999: Specialize loading attributes from modules in free-threaded builds (#127711) We use the same approach that was used for specialization of LOAD_GLOBAL in free-threaded builds: _CHECK_ATTR_MODULE is renamed to _CHECK_ATTR_MODULE_PUSH_KEYS; it pushes the keys object for the following _LOAD_ATTR_MODULE_FROM_KEYS (nee _LOAD_ATTR_MODULE). This arrangement avoids having to recheck the keys version. _LOAD_ATTR_MODULE is renamed to _LOAD_ATTR_MODULE_FROM_KEYS; it loads the value from the keys object pushed by the preceding _CHECK_ATTR_MODULE_PUSH_KEYS at the cached index. --- Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_uop_ids.h | 141 +++++++++++---------- Include/internal/pycore_uop_metadata.h | 16 ++- Lib/test/test_capi/test_misc.py | 47 +++++-- Lib/test/test_generated_cases.py | 74 +++++++++++ Lib/test/test_opcache.py | 2 +- Python/bytecodes.c | 59 ++++++--- Python/executor_cases.c.h | 70 ++++++++-- Python/generated_cases.c.h | 33 +++-- Python/optimizer_analysis.c | 2 +- Python/optimizer_bytecodes.c | 10 +- Python/optimizer_cases.c.h | 47 +++++-- Python/specialize.c | 92 ++++++++------ Tools/cases_generator/generators_common.py | 15 +++ Tools/cases_generator/stack.py | 4 + 15 files changed, 437 insertions(+), 179 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 81dde66a6f26c2..28aa1120414337 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1492,7 +1492,7 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { return 0; } case LOAD_ATTR_MODULE: { - *effect = Py_MAX(0, (oparg & 1)); + *effect = Py_MAX(1, (oparg & 1)); return 0; } case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { @@ -2271,7 +2271,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, + [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE_PUSH_KEYS, 2, 1 }, { _LOAD_ATTR_MODULE_FROM_KEYS, 1, 3 } } }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_PROPERTY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_PROPERTY_FRAME, 4, 5 }, { _SAVE_RETURN_OFFSET, 7, 9 }, { _PUSH_FRAME, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index fab4ce6a25b347..45563585dd5681 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -55,7 +55,7 @@ extern "C" { #define _CHECK_AND_ALLOCATE_OBJECT 327 #define _CHECK_ATTR_CLASS 328 #define _CHECK_ATTR_METHOD_LAZY_DICT 329 -#define _CHECK_ATTR_MODULE 330 +#define _CHECK_ATTR_MODULE_PUSH_KEYS 330 #define _CHECK_ATTR_WITH_HINT 331 #define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 332 #define _CHECK_EG_MATCH CHECK_EG_MATCH @@ -186,115 +186,116 @@ extern "C" { #define _LOAD_ATTR_METHOD_NO_DICT 416 #define _LOAD_ATTR_METHOD_WITH_VALUES 417 #define _LOAD_ATTR_MODULE 418 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 419 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 420 -#define _LOAD_ATTR_PROPERTY_FRAME 421 -#define _LOAD_ATTR_SLOT 422 -#define _LOAD_ATTR_SLOT_0 423 -#define _LOAD_ATTR_SLOT_1 424 -#define _LOAD_ATTR_WITH_HINT 425 +#define _LOAD_ATTR_MODULE_FROM_KEYS 419 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 420 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 421 +#define _LOAD_ATTR_PROPERTY_FRAME 422 +#define _LOAD_ATTR_SLOT 423 +#define _LOAD_ATTR_SLOT_0 424 +#define _LOAD_ATTR_SLOT_1 425 +#define _LOAD_ATTR_WITH_HINT 426 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 426 +#define _LOAD_BYTECODE 427 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 427 -#define _LOAD_CONST_INLINE_BORROW 428 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 429 -#define _LOAD_CONST_INLINE_WITH_NULL 430 +#define _LOAD_CONST_INLINE 428 +#define _LOAD_CONST_INLINE_BORROW 429 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 430 +#define _LOAD_CONST_INLINE_WITH_NULL 431 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 431 -#define _LOAD_FAST_0 432 -#define _LOAD_FAST_1 433 -#define _LOAD_FAST_2 434 -#define _LOAD_FAST_3 435 -#define _LOAD_FAST_4 436 -#define _LOAD_FAST_5 437 -#define _LOAD_FAST_6 438 -#define _LOAD_FAST_7 439 +#define _LOAD_FAST 432 +#define _LOAD_FAST_0 433 +#define _LOAD_FAST_1 434 +#define _LOAD_FAST_2 435 +#define _LOAD_FAST_3 436 +#define _LOAD_FAST_4 437 +#define _LOAD_FAST_5 438 +#define _LOAD_FAST_6 439 +#define _LOAD_FAST_7 440 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 440 -#define _LOAD_GLOBAL_BUILTINS 441 -#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 442 -#define _LOAD_GLOBAL_MODULE 443 -#define _LOAD_GLOBAL_MODULE_FROM_KEYS 444 +#define _LOAD_GLOBAL 441 +#define _LOAD_GLOBAL_BUILTINS 442 +#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 443 +#define _LOAD_GLOBAL_MODULE 444 +#define _LOAD_GLOBAL_MODULE_FROM_KEYS 445 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 445 -#define _LOAD_SMALL_INT_0 446 -#define _LOAD_SMALL_INT_1 447 -#define _LOAD_SMALL_INT_2 448 -#define _LOAD_SMALL_INT_3 449 +#define _LOAD_SMALL_INT 446 +#define _LOAD_SMALL_INT_0 447 +#define _LOAD_SMALL_INT_1 448 +#define _LOAD_SMALL_INT_2 449 +#define _LOAD_SMALL_INT_3 450 #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 450 +#define _MAKE_CALLARGS_A_TUPLE 451 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 451 +#define _MAKE_WARM 452 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 452 -#define _MAYBE_EXPAND_METHOD_KW 453 -#define _MONITOR_CALL 454 -#define _MONITOR_JUMP_BACKWARD 455 -#define _MONITOR_RESUME 456 +#define _MAYBE_EXPAND_METHOD 453 +#define _MAYBE_EXPAND_METHOD_KW 454 +#define _MONITOR_CALL 455 +#define _MONITOR_JUMP_BACKWARD 456 +#define _MONITOR_RESUME 457 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 457 -#define _POP_JUMP_IF_TRUE 458 +#define _POP_JUMP_IF_FALSE 458 +#define _POP_JUMP_IF_TRUE 459 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 459 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 460 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 460 +#define _PUSH_FRAME 461 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 461 -#define _PY_FRAME_KW 462 -#define _QUICKEN_RESUME 463 -#define _REPLACE_WITH_TRUE 464 +#define _PY_FRAME_GENERAL 462 +#define _PY_FRAME_KW 463 +#define _QUICKEN_RESUME 464 +#define _REPLACE_WITH_TRUE 465 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 465 -#define _SEND 466 -#define _SEND_GEN_FRAME 467 +#define _SAVE_RETURN_OFFSET 466 +#define _SEND 467 +#define _SEND_GEN_FRAME 468 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 468 -#define _STORE_ATTR 469 -#define _STORE_ATTR_INSTANCE_VALUE 470 -#define _STORE_ATTR_SLOT 471 -#define _STORE_ATTR_WITH_HINT 472 +#define _START_EXECUTOR 469 +#define _STORE_ATTR 470 +#define _STORE_ATTR_INSTANCE_VALUE 471 +#define _STORE_ATTR_SLOT 472 +#define _STORE_ATTR_WITH_HINT 473 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 473 -#define _STORE_FAST_0 474 -#define _STORE_FAST_1 475 -#define _STORE_FAST_2 476 -#define _STORE_FAST_3 477 -#define _STORE_FAST_4 478 -#define _STORE_FAST_5 479 -#define _STORE_FAST_6 480 -#define _STORE_FAST_7 481 +#define _STORE_FAST 474 +#define _STORE_FAST_0 475 +#define _STORE_FAST_1 476 +#define _STORE_FAST_2 477 +#define _STORE_FAST_3 478 +#define _STORE_FAST_4 479 +#define _STORE_FAST_5 480 +#define _STORE_FAST_6 481 +#define _STORE_FAST_7 482 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 482 -#define _STORE_SUBSCR 483 +#define _STORE_SLICE 483 +#define _STORE_SUBSCR 484 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 484 -#define _TO_BOOL 485 +#define _TIER2_RESUME_CHECK 485 +#define _TO_BOOL 486 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -304,13 +305,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 486 +#define _UNPACK_SEQUENCE 487 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 486 +#define MAX_UOP_ID 487 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 89fce193f40bd8..dd775d3f7d3cdd 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -152,8 +152,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, - [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG, - [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CHECK_ATTR_MODULE_PUSH_KEYS] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_MODULE_FROM_KEYS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CHECK_ATTR_WITH_HINT] = HAS_EXIT_FLAG, [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, @@ -283,6 +283,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_START_EXECUTOR] = 0, @@ -346,7 +347,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", - [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", + [_CHECK_ATTR_MODULE_PUSH_KEYS] = "_CHECK_ATTR_MODULE_PUSH_KEYS", [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", @@ -459,6 +460,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_LOAD_ATTR_MODULE_FROM_KEYS] = "_LOAD_ATTR_MODULE_FROM_KEYS", [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", @@ -845,10 +847,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_INSTANCE_VALUE: return 1; - case _CHECK_ATTR_MODULE: + case _CHECK_ATTR_MODULE_PUSH_KEYS: return 0; - case _LOAD_ATTR_MODULE: - return 1; + case _LOAD_ATTR_MODULE_FROM_KEYS: + return 2; case _CHECK_ATTR_WITH_HINT: return 0; case _LOAD_ATTR_WITH_HINT: @@ -1107,6 +1109,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_GLOBAL_BUILTINS: return 0; + case _LOAD_ATTR_MODULE: + return 1; case _INTERNAL_INCREMENT_OPT_COUNTER: return 1; case _DYNAMIC_EXIT: diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 61512e610f46f2..ada30181aeeca9 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -48,6 +48,8 @@ # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +from _testcapi import HeapCTypeSubclass, HeapCTypeSubclassWithFinalizer + import _testlimitedcapi import _testinternalcapi @@ -653,9 +655,9 @@ def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self): self.assertEqual(type_refcnt - 1, sys.getrefcount(_testcapi.HeapCTypeSubclass)) def test_c_subclass_of_heap_ctype_with_del_modifying_dunder_class_only_decrefs_once(self): - subclass_instance = _testcapi.HeapCTypeSubclassWithFinalizer() - type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclassWithFinalizer) - new_type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass) + subclass_instance = HeapCTypeSubclassWithFinalizer() + type_refcnt = sys.getrefcount(HeapCTypeSubclassWithFinalizer) + new_type_refcnt = sys.getrefcount(HeapCTypeSubclass) # Test that subclass instance was fully created self.assertEqual(subclass_instance.value, 10) @@ -665,19 +667,46 @@ def test_c_subclass_of_heap_ctype_with_del_modifying_dunder_class_only_decrefs_o del subclass_instance # Test that setting __class__ modified the reference counts of the types + # + # This is highly sensitive to implementation details and may break in the future. + # + # We expect the refcount on the old type, HeapCTypeSubclassWithFinalizer, to + # remain the same: the finalizer gets a strong reference (+1) when it gets the + # type from the module and setting __class__ decrements the refcount (-1). + # + # We expect the refcount on the new type, HeapCTypeSubclass, to increase by 2: + # the finalizer get a strong reference (+1) when it gets the type from the + # module and setting __class__ increments the refcount (+1). + expected_type_refcnt = type_refcnt + expected_new_type_refcnt = new_type_refcnt + 2 + + if not Py_GIL_DISABLED: + # In default builds the result returned from sys.getrefcount + # includes a temporary reference that is created by the interpreter + # when it pushes its argument on the operand stack. This temporary + # reference is not included in the result returned by Py_REFCNT, which + # is used in the finalizer. + # + # In free-threaded builds the result returned from sys.getrefcount + # does not include the temporary reference. Types use deferred + # refcounting and the interpreter will not create a new reference + # for deferred values on the operand stack. + expected_type_refcnt -= 1 + expected_new_type_refcnt -= 1 + if support.Py_DEBUG: # gh-89373: In debug mode, _Py_Dealloc() keeps a strong reference # to the type while calling tp_dealloc() - self.assertEqual(type_refcnt, _testcapi.HeapCTypeSubclassWithFinalizer.refcnt_in_del) - else: - self.assertEqual(type_refcnt - 1, _testcapi.HeapCTypeSubclassWithFinalizer.refcnt_in_del) - self.assertEqual(new_type_refcnt + 1, _testcapi.HeapCTypeSubclass.refcnt_in_del) + expected_type_refcnt += 1 + + self.assertEqual(expected_type_refcnt, HeapCTypeSubclassWithFinalizer.refcnt_in_del) + self.assertEqual(expected_new_type_refcnt, HeapCTypeSubclass.refcnt_in_del) # Test that the original type already has decreased its refcnt - self.assertEqual(type_refcnt - 1, sys.getrefcount(_testcapi.HeapCTypeSubclassWithFinalizer)) + self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapCTypeSubclassWithFinalizer)) # Test that subtype_dealloc decref the newly assigned __class__ only once - self.assertEqual(new_type_refcnt, sys.getrefcount(_testcapi.HeapCTypeSubclass)) + self.assertEqual(new_type_refcnt, sys.getrefcount(HeapCTypeSubclass)) def test_heaptype_with_setattro(self): obj = _testcapi.HeapCTypeSetattr() diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 66862ec17cca98..9c65e81dfe4be1 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1639,6 +1639,80 @@ def test_escaping_call_next_to_cmacro(self): """ self.run_cases_test(input, output) + def test_pop_dead_inputs_all_live(self): + input = """ + inst(OP, (a, b --)) { + POP_DEAD_INPUTS(); + HAM(a, b); + INPUTS_DEAD(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef a; + _PyStackRef b; + b = stack_pointer[-1]; + a = stack_pointer[-2]; + HAM(a, b); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pop_dead_inputs_some_live(self): + input = """ + inst(OP, (a, b, c --)) { + POP_DEAD_INPUTS(); + HAM(a); + INPUTS_DEAD(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef a; + a = stack_pointer[-3]; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + HAM(a); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pop_dead_inputs_with_output(self): + input = """ + inst(OP, (a, b -- c)) { + POP_DEAD_INPUTS(); + c = SPAM(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef c; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + c = SPAM(); + stack_pointer[0] = c; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 50b5f365165921..0a7557adc4763b 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -892,7 +892,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_WITH_VALUES" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_module(self): def get_items(): items = [] diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f0eb5405faeff5..772b46d17ec198 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2070,7 +2070,7 @@ dummy_func( }; specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; @@ -2079,7 +2079,7 @@ dummy_func( } OPCODE_DEFERRED_INC(LOAD_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { @@ -2158,33 +2158,43 @@ dummy_func( _LOAD_ATTR_INSTANCE_VALUE + unused/5; // Skip over rest of cache - op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { + op(_CHECK_ATTR_MODULE_PUSH_KEYS, (dict_version/2, owner -- owner, mod_keys: PyDictKeysObject *)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != dict_version); - } - - op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - PyObject *attr_o = ep->me_value; + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version); + mod_keys = keys; + } + + op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys: PyDictKeysObject * -- attr, null if (oparg & 1))) { + assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + DEAD(mod_keys); + // Clear mod_keys from stack in case we need to deopt + POP_DEAD_INPUTS(); DEOPT_IF(attr_o == NULL); - STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + DEOPT_IF(true); + } + #else Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); } macro(LOAD_ATTR_MODULE) = unused/1 + - _CHECK_ATTR_MODULE + - _LOAD_ATTR_MODULE + + _CHECK_ATTR_MODULE_PUSH_KEYS + + _LOAD_ATTR_MODULE_FROM_KEYS + unused/5; op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { @@ -4963,6 +4973,21 @@ dummy_func( null = PyStackRef_NULL; } + tier2 op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + PyObject *attr_o = ep->me_value; + DEOPT_IF(attr_o == NULL); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + DECREF_INPUTS(); + } + /* Internal -- for testing executors */ op(_INTERNAL_INCREMENT_OPT_COUNTER, (opt --)) { _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 19ba67a8af6769..55e9c3aa2db64d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2641,8 +2641,9 @@ /* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */ - case _CHECK_ATTR_MODULE: { + case _CHECK_ATTR_MODULE_PUSH_KEYS: { _PyStackRef owner; + PyDictKeysObject *mod_keys; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -2652,33 +2653,51 @@ } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); - if (dict->ma_keys->dk_version != dict_version) { + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + mod_keys = keys; + stack_pointer[0].bits = (uintptr_t)mod_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_ATTR_MODULE: { + case _LOAD_ATTR_MODULE_FROM_KEYS: { + PyDictKeysObject *mod_keys; _PyStackRef owner; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + mod_keys = (PyDictKeysObject *)stack_pointer[-1].bits; + owner = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - PyObject *attr_o = ep->me_value; + assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + // Clear mod_keys from stack in case we need to deopt + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #else Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; @@ -5928,6 +5947,35 @@ break; } + case _LOAD_ATTR_MODULE: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + PyObject *attr_o = ep->me_value; + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _INTERNAL_INCREMENT_OPT_COUNTER: { _PyStackRef opt; opt = stack_pointer[-1]; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 51227c9868b8cc..94343f953221eb 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5200,7 +5200,7 @@ owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; @@ -5211,7 +5211,7 @@ } OPCODE_DEFERRED_INC(LOAD_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 8 cache entries */ // _LOAD_ATTR @@ -5553,10 +5553,11 @@ INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; + PyDictKeysObject *mod_keys; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE + // _CHECK_ATTR_MODULE_PUSH_KEYS { owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); @@ -5564,21 +5565,29 @@ DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + mod_keys = keys; } - // _LOAD_ATTR_MODULE + // _LOAD_ATTR_MODULE_FROM_KEYS { uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - PyObject *attr_o = ep->me_value; + assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + // Clear mod_keys from stack in case we need to deopt DEOPT_IF(attr_o == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + DEOPT_IF(true, LOAD_ATTR); + } + #else Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a4a0472b64e57c..0ef15c630e91db 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -95,7 +95,7 @@ type_watcher_callback(PyTypeObject* type) static PyObject * convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) { - assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE); + assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE_FROM_KEYS); assert(PyDict_CheckExact(obj)); PyDictObject *dict = (PyDictObject *)obj; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 42bdbd9ca8d0cd..0b8aff02367e31 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -492,8 +492,9 @@ dummy_func(void) { (void)owner; } - op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { + op(_CHECK_ATTR_MODULE_PUSH_KEYS, (dict_version/2, owner -- owner, mod_keys)) { (void)dict_version; + mod_keys = sym_new_not_null(ctx); if (sym_is_const(owner)) { PyObject *cnst = sym_get_const(owner); if (PyModule_CheckExact(cnst)) { @@ -515,12 +516,12 @@ dummy_func(void) { self_or_null = sym_new_unknown(ctx); } - op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr, null if (oparg & 1))) { (void)index; null = sym_new_null(ctx); attr = NULL; if (this_instr[-1].opcode == _NOP) { - // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched. assert(sym_is_const(owner)); PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); assert(PyModule_CheckExact(mod)); @@ -530,6 +531,9 @@ dummy_func(void) { this_instr[-1].opcode = _POP_TOP; attr = sym_new_const(ctx, res); } + else { + this_instr->opcode = _LOAD_ATTR_MODULE; + } } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f77a5aa35bdf82..f4fbe8c8aa0480 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1134,61 +1134,74 @@ break; } - case _CHECK_ATTR_MODULE: { + case _CHECK_ATTR_MODULE_PUSH_KEYS: { _Py_UopsSymbol *owner; + _Py_UopsSymbol *mod_keys; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand0; (void)dict_version; + mod_keys = sym_new_not_null(ctx); if (sym_is_const(owner)) { PyObject *cnst = sym_get_const(owner); if (PyModule_CheckExact(cnst)) { PyModuleObject *mod = (PyModuleObject *)cnst; PyObject *dict = mod->md_dict; + stack_pointer[0] = mod_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); uint64_t watched_mutations = get_mutations(dict); if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); this_instr->opcode = _NOP; } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } } + stack_pointer[0] = mod_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_ATTR_MODULE: { + case _LOAD_ATTR_MODULE_FROM_KEYS: { _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; - owner = stack_pointer[-1]; + owner = stack_pointer[-2]; uint16_t index = (uint16_t)this_instr->operand0; (void)index; null = sym_new_null(ctx); attr = NULL; if (this_instr[-1].opcode == _NOP) { - // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched. assert(sym_is_const(owner)); PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); assert(PyModule_CheckExact(mod)); PyObject *dict = mod->md_dict; - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + stack_pointer[-2] = attr; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { this_instr[-1].opcode = _POP_TOP; attr = sym_new_const(ctx, res); } - stack_pointer += -(oparg & 1); + else { + this_instr->opcode = _LOAD_ATTR_MODULE; + } + stack_pointer += 1 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + stack_pointer[-2] = attr; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -2528,6 +2541,18 @@ break; } + case _LOAD_ATTR_MODULE: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _INTERNAL_INCREMENT_OPT_COUNTER: { stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/specialize.c b/Python/specialize.c index fd182e7d7a9215..6eb298217ec2d3 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -738,22 +738,16 @@ unspecialize(_Py_CODEUNIT *instr) } static int function_kind(PyCodeObject *code); +#ifndef Py_GIL_DISABLED static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); +#endif static uint32_t type_get_version(PyTypeObject *t, int opcode); static int -specialize_module_load_attr( - PyObject *owner, _Py_CODEUNIT *instr, PyObject *name -) { +specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name) +{ _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - PyModuleObject *m = (PyModuleObject *)owner; - assert((Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); - PyDictObject *dict = (PyDictObject *)m->md_dict; - if (dict == NULL) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT); - return -1; - } if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING); return -1; @@ -773,19 +767,35 @@ specialize_module_load_attr( SPEC_FAIL_OUT_OF_RANGE); return -1; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( - _PyInterpreterState_GET(), dict->ma_keys); + uint32_t keys_version = _PyDict_GetKeysVersionForCurrentState( + _PyInterpreterState_GET(), dict); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->version, keys_version); cache->index = (uint16_t)index; - instr->op.code = LOAD_ATTR_MODULE; + specialize(instr, LOAD_ATTR_MODULE); return 0; } - +static int +specialize_module_load_attr( + PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) +{ + PyModuleObject *m = (PyModuleObject *)owner; + assert((Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); + PyDictObject *dict = (PyDictObject *)m->md_dict; + if (dict == NULL) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT); + return -1; + } + int result; + Py_BEGIN_CRITICAL_SECTION(dict); + result = specialize_module_load_attr_lock_held(dict, instr, name); + Py_END_CRITICAL_SECTION(); + return result; +} /* Attribute specialization */ @@ -968,7 +978,7 @@ specialize_dict_access( } write_u32(cache->version, type->tp_version_tag); cache->index = (uint16_t)offset; - instr->op.code = values_op; + specialize(instr, values_op); } else { PyDictObject *dict = _PyObject_GetManagedDict(owner); @@ -992,11 +1002,12 @@ specialize_dict_access( } cache->index = (uint16_t)index; write_u32(cache->version, type->tp_version_tag); - instr->op.code = hint_op; + specialize(instr, hint_op); } return 1; } +#ifndef Py_GIL_DISABLED static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, PyObject* descr, DescriptorClassification kind, bool is_method); static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name); @@ -1093,7 +1104,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na write_u32(lm_cache->type_version, type->tp_version_tag); /* borrowed */ write_obj(lm_cache->descr, fget); - instr->op.code = LOAD_ATTR_PROPERTY; + specialize(instr, LOAD_ATTR_PROPERTY); return 0; } case OBJECT_SLOT: @@ -1117,7 +1128,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - instr->op.code = LOAD_ATTR_SLOT; + specialize(instr, LOAD_ATTR_SLOT); return 0; } case DUNDER_CLASS: @@ -1126,7 +1137,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - instr->op.code = LOAD_ATTR_SLOT; + specialize(instr, LOAD_ATTR_SLOT); return 0; } case OTHER_SLOT: @@ -1162,7 +1173,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na /* borrowed */ write_obj(lm_cache->descr, descr); write_u32(lm_cache->type_version, type->tp_version_tag); - instr->op.code = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); return 0; } case BUILTIN_CLASSMETHOD: @@ -1186,6 +1197,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (shadow) { goto try_instance; } + set_counter((_Py_BackoffCounter*)instr + 1, adaptive_counter_cooldown()); return 0; } Py_UNREACHABLE(); @@ -1197,14 +1209,14 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na } return -1; } +#endif // Py_GIL_DISABLED void _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) { - _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *type = Py_TYPE(owner); bool fail; @@ -1219,22 +1231,24 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam fail = specialize_module_load_attr(owner, instr, name); } else if (PyType_Check(owner)) { + #ifdef Py_GIL_DISABLED + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); + fail = true; + #else fail = specialize_class_load_attr(owner, instr, name); + #endif } else { + #ifdef Py_GIL_DISABLED + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); + fail = true; + #else fail = specialize_instance_load_attr(owner, instr, name); + #endif } if (fail) { - STAT_INC(LOAD_ATTR, failure); - assert(!PyErr_Occurred()); - instr->op.code = LOAD_ATTR; - cache->counter = adaptive_counter_backoff(cache->counter); - } - else { - STAT_INC(LOAD_ATTR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr); } } @@ -1339,6 +1353,7 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na cache->counter = adaptive_counter_cooldown(); } +#ifndef Py_GIL_DISABLED #ifdef Py_STATS static int @@ -1422,10 +1437,10 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, write_obj(cache->descr, descr); if (metaclass_check) { write_u32(cache->keys_version, Py_TYPE(cls)->tp_version_tag); - instr->op.code = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); } else { - instr->op.code = LOAD_ATTR_CLASS; + specialize(instr, LOAD_ATTR_CLASS); } return 0; #ifdef Py_STATS @@ -1461,7 +1476,7 @@ PyObject *descr, DescriptorClassification kind, bool is_method) return 0; } write_u32(cache->keys_version, keys_version); - instr->op.code = is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); } else { Py_ssize_t dictoffset; @@ -1476,7 +1491,7 @@ PyObject *descr, DescriptorClassification kind, bool is_method) } } if (dictoffset == 0) { - instr->op.code = is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + specialize(instr, is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT); } else if (is_method) { PyObject *dict = *(PyObject **) ((char *)owner + dictoffset); @@ -1490,7 +1505,7 @@ PyObject *descr, DescriptorClassification kind, bool is_method) dictoffset -= MANAGED_DICT_OFFSET; assert(((uint16_t)dictoffset) == dictoffset); cache->dict_offset = (uint16_t)dictoffset; - instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT; + specialize(instr, LOAD_ATTR_METHOD_LAZY_DICT); } else { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); @@ -1516,6 +1531,9 @@ PyObject *descr, DescriptorClassification kind, bool is_method) return 1; } +#endif // Py_GIL_DISABLED + + static void specialize_load_global_lock_held( PyObject *globals, PyObject *builtins, @@ -1661,6 +1679,7 @@ function_kind(PyCodeObject *code) { return SIMPLE_FUNCTION; } +#ifndef Py_GIL_DISABLED /* Returning false indicates a failure. */ static bool function_check_args(PyObject *o, int expected_argcount, int opcode) @@ -1693,6 +1712,7 @@ function_get_version(PyObject *o, int opcode) } return version; } +#endif // Py_GIL_DISABLED /* Returning 0 indicates a failure. */ static uint32_t diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index dad2557e97a948..d17617cab0266b 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -120,6 +120,7 @@ def __init__(self, out: CWriter): "PyStackRef_AsPyObjectSteal": self.stackref_steal, "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, + "POP_DEAD_INPUTS": self.pop_dead_inputs, } self.out = out @@ -348,6 +349,20 @@ def save_stack( self.emit_save(storage) return True + def pop_dead_inputs( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + storage.pop_dead_inputs(self.out) + return True + def emit_reload(self, storage: Storage) -> None: storage.reload(self.out) self._print_storage(storage) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 286f47d0cfb11b..9471fe0e56f7d8 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -512,6 +512,10 @@ def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = self._push_defined_outputs() self.stack.flush(out, cast_type, extract_bits) + def pop_dead_inputs(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = True) -> None: + self.clear_dead_inputs() + self.stack.flush(out, cast_type, extract_bits) + def save(self, out: CWriter) -> None: assert self.spilled >= 0 if self.spilled == 0: From c0264fc57c51e68015bef95a2208711356b57c1f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Fri, 13 Dec 2024 23:36:47 -0800 Subject: [PATCH 203/427] gh-127747: Resolve BytesWarning in test.support.strace_helper (#127849) The strace_helper code has a _make_error function to simplify making StraceResult objects in error cases. That takes a details parameter which is either a caught OSError or `bytes`. If it's bytes, _make_error would implicitly coerce that to a str inside of a f-string, resulting in a BytesWarning. It's useful to see if it's an OSError or bytes when debugging, resolve by changing to format with repr(). This is an error message on an internal helper. A non-zero exit code occurs if the strace binary isn't found, and no events will be parsed in that case (there is no output). Handle that case by checking exit code before checking for events. Still asserting around events rather than returning false, so that hopefully if there's some change to `strace` that breaks the parsing, will see that as a test failure rather than silently loosing strace tests because they are auto-disabled. --- Lib/test/support/strace_helper.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py index eab16ea3e2889f..798d6c6886962f 100644 --- a/Lib/test/support/strace_helper.py +++ b/Lib/test/support/strace_helper.py @@ -104,7 +104,7 @@ def _make_error(reason, details): return StraceResult( strace_returncode=-1, python_returncode=-1, - event_bytes=f"error({reason},details={details}) = -1".encode('utf-8'), + event_bytes= f"error({reason},details={details!r}) = -1".encode('utf-8'), stdout=res.out if res else b"", stderr=res.err if res else b"") @@ -179,9 +179,10 @@ def get_syscalls(code, strace_flags, prelude="", cleanup="", @cache def _can_strace(): res = strace_python("import sys; sys.exit(0)", [], check=False) - assert res.events(), "Should have parsed multiple calls" - - return res.strace_returncode == 0 and res.python_returncode == 0 + if res.strace_returncode == 0 and res.python_returncode == 0: + assert res.events(), "Should have parsed multiple calls" + return True + return False def requires_strace(): From 78e766f2e217572eacefba9ec31396b016aa88c2 Mon Sep 17 00:00:00 2001 From: Totosuki <116938397+totosuki@users.noreply.github.com> Date: Sat, 14 Dec 2024 16:38:54 +0900 Subject: [PATCH 204/427] Fix typo in docstring: quadruple double quotes (#127913) --- Lib/test/test_ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 59f37b3f9a7575..3f6f890bbdc658 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -151,7 +151,7 @@ def is_ubuntu(): if is_ubuntu(): def seclevel_workaround(*ctxs): - """"Lower security level to '1' and allow all ciphers for TLS 1.0/1""" + """Lower security level to '1' and allow all ciphers for TLS 1.0/1""" for ctx in ctxs: if ( hasattr(ctx, "minimum_version") and From e2325c9db0650fc06d909eb2b5930c0573f24f71 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Sat, 14 Dec 2024 16:28:26 +0300 Subject: [PATCH 205/427] gh-127852: add remark about ',' separator (#127854) Specify that it is valid for floats and ints with 'd' presentation and an error otherwise. Co-authored-by: Terry Jan Reedy --------- Co-authored-by: Terry Jan Reedy --- Doc/library/string.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index a000bb49f14800..913672a3ff2270 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -409,7 +409,9 @@ conversions, trailing zeros are not removed from the result. .. index:: single: , (comma); in string formatting -The ``','`` option signals the use of a comma for a thousands separator. +The ``','`` option signals the use of a comma for a thousands separator for +floating-point presentation types and for integer presentation type ``'d'``. +For other presentation types, this option is an error. For a locale aware separator, use the ``'n'`` integer presentation type instead. From 0ac40acec045c4ce780cf7d887fcbe4c661e82b7 Mon Sep 17 00:00:00 2001 From: Andrey Efremov Date: Sat, 14 Dec 2024 22:25:49 +0700 Subject: [PATCH 206/427] gh-127353: Allow to force color output on Windows V2 (#127926) --- Lib/_colorize.py | 17 ++--- Lib/test/test__colorize.py | 70 +++++++++++-------- ...-11-28-15-55-48.gh-issue-127353.i-XOXg.rst | 2 + 3 files changed, 51 insertions(+), 38 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 845fb57a90abb8..709081e25ec59b 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -32,14 +32,6 @@ def get_colors(colorize: bool = False) -> ANSIColors: def can_colorize() -> bool: - if sys.platform == "win32": - try: - import nt - - if not nt._supports_virtual_terminal(): - return False - except (ImportError, AttributeError): - return False if not sys.flags.ignore_environment: if os.environ.get("PYTHON_COLORS") == "0": return False @@ -58,6 +50,15 @@ def can_colorize() -> bool: if not hasattr(sys.stderr, "fileno"): return False + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + try: return os.isatty(sys.stderr.fileno()) except io.UnsupportedOperation: diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index d55b97ade68cef..1871775fa205a2 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -19,40 +19,50 @@ def tearDownModule(): class TestColorizeFunction(unittest.TestCase): @force_not_colorized def test_colorized_detection_checks_for_environment_variables(self): - if sys.platform == "win32": - virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", - return_value=True) - else: - virtual_patching = contextlib.nullcontext() - with virtual_patching: - - flags = unittest.mock.MagicMock(ignore_environment=False) - with (unittest.mock.patch("os.isatty") as isatty_mock, - unittest.mock.patch("sys.flags", flags), - unittest.mock.patch("_colorize.can_colorize", ORIGINAL_CAN_COLORIZE)): - isatty_mock.return_value = True - with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): - self.assertEqual(_colorize.can_colorize(), False) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): - self.assertEqual(_colorize.can_colorize(), True) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): - self.assertEqual(_colorize.can_colorize(), False) - with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): + flags = unittest.mock.MagicMock(ignore_environment=False) + with (unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.stderr") as stderr_mock, + unittest.mock.patch("sys.flags", flags), + unittest.mock.patch("_colorize.can_colorize", ORIGINAL_CAN_COLORIZE), + (unittest.mock.patch("nt._supports_virtual_terminal", return_value=False) + if sys.platform == "win32" else + contextlib.nullcontext()) as vt_mock): + + isatty_mock.return_value = True + stderr_mock.fileno.return_value = 2 + stderr_mock.isatty.return_value = True + with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): + self.assertEqual(_colorize.can_colorize(), False) + + with unittest.mock.patch("os.environ", {}): + if sys.platform == "win32": self.assertEqual(_colorize.can_colorize(), False) - with unittest.mock.patch("os.environ", - {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): + + vt_mock.return_value = True self.assertEqual(_colorize.can_colorize(), True) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): + else: self.assertEqual(_colorize.can_colorize(), True) - with unittest.mock.patch("os.environ", - {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): - self.assertEqual(_colorize.can_colorize(), False) - with unittest.mock.patch("os.environ", - {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): - self.assertEqual(_colorize.can_colorize(), False) + isatty_mock.return_value = False - with unittest.mock.patch("os.environ", {}): - self.assertEqual(_colorize.can_colorize(), False) + stderr_mock.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(), False) if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst b/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst new file mode 100644 index 00000000000000..88661b9a611071 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst @@ -0,0 +1,2 @@ +Allow to force color output on Windows using environment variables. Patch by +Andrey Efremov. From 7900a85019457c14e8c6abac532846bc9f26760d Mon Sep 17 00:00:00 2001 From: Steve C Date: Sun, 15 Dec 2024 07:28:43 -0500 Subject: [PATCH 207/427] Clarify ast docs to use a less confusing example for `ast.ParamSpec` (#127955) Fix typo in ast docs: ParamSpec defaults --- Doc/library/ast.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 22d8c87cb58e78..fd901e232855b5 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1807,7 +1807,7 @@ aliases. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[**P = (int, str)] = Callable[P, int]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[**P = [int, str]] = Callable[P, int]"), indent=4)) Module( body=[ TypeAlias( @@ -1815,7 +1815,7 @@ aliases. type_params=[ ParamSpec( name='P', - default_value=Tuple( + default_value=List( elts=[ Name(id='int', ctx=Load()), Name(id='str', ctx=Load())], From ab05beb8cea62636bd86f6f7cf1a82d7efca7162 Mon Sep 17 00:00:00 2001 From: Ed Nutting Date: Sun, 15 Dec 2024 14:51:03 +0100 Subject: [PATCH 208/427] gh-127599: Fix _Py_RefcntAdd missing calls to _Py_INCREF_STAT_INC/_Py_INCREF_IMMORTAL_STAT_INC (#127717) Previously, `_Py_RefcntAdd` hasn't called `_Py_INCREF_STAT_INC/_Py_INCREF_IMMORTAL_STAT_INC` which is incorrect. Now it has been fixed. --- Include/cpython/pystats.h | 6 ++++++ Include/internal/pycore_object.h | 5 +++++ .../2024-12-07-13-06-09.gh-issue-127599.tXCZb_.rst | 2 ++ 3 files changed, 13 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-07-13-06-09.gh-issue-127599.tXCZb_.rst diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 2ae48002d720e9..29ef0c0e4d4e72 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -18,6 +18,12 @@ // // Define _PY_INTERPRETER macro to increment interpreter_increfs and // interpreter_decrefs. Otherwise, increment increfs and decrefs. +// +// The number of incref operations counted by `incref` and +// `interpreter_incref` is the number of increment operations, which is +// not equal to the total of all reference counts. A single increment +// operation may increase the reference count of an object by more than +// one. For example, see `_Py_RefcntAdd`. #ifndef Py_CPYTHON_PYSTATS_H # error "this header file must not be included directly" diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 668ea47ca727e2..d7d68f938a9f0a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -131,6 +131,7 @@ extern void _Py_DecRefTotal(PyThreadState *); static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) { if (_Py_IsImmortal(op)) { + _Py_INCREF_IMMORTAL_STAT_INC(); return; } #ifdef Py_REF_DEBUG @@ -159,6 +160,10 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) _Py_atomic_add_ssize(&op->ob_ref_shared, (n << _Py_REF_SHARED_SHIFT)); } #endif + // Although the ref count was increased by `n` (which may be greater than 1) + // it is only a single increment (i.e. addition) operation, so only 1 refcnt + // increment operation is counted. + _Py_INCREF_STAT_INC(); } #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-07-13-06-09.gh-issue-127599.tXCZb_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-07-13-06-09.gh-issue-127599.tXCZb_.rst new file mode 100644 index 00000000000000..565ecb82c71926 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-07-13-06-09.gh-issue-127599.tXCZb_.rst @@ -0,0 +1,2 @@ +Fix statistics for increments of object reference counts (in particular, when +a reference count was increased by more than 1 in a single operation). From 3683b2f9e5972a2feb67a051fcf898a1b58a54fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Sun, 15 Dec 2024 15:40:19 +0000 Subject: [PATCH 209/427] getpath: Add comments highlighing details of the pyvenv.cfg detection (#127966) --- Modules/getpath.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Modules/getpath.py b/Modules/getpath.py index 7949fd813d0d07..b14f985a0d5f97 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -363,10 +363,20 @@ def search_up(prefix, *landmarks, test=isfile): venv_prefix = None pyvenvcfg = [] + # Search for the 'home' key in pyvenv.cfg. Currently, we don't consider the + # presence of a pyvenv.cfg file without a 'home' key to signify the + # existence of a virtual environment — we quietly ignore them. + # XXX: If we don't find a 'home' key, we don't look for another pyvenv.cfg! for line in pyvenvcfg: key, had_equ, value = line.partition('=') if had_equ and key.strip().lower() == 'home': + # Override executable_dir/real_executable_dir with the value from 'home'. + # These values may be later used to calculate prefix/base_prefix, if a more + # reliable source — like the runtime library (libpython) path — isn't available. executable_dir = real_executable_dir = value.strip() + # If base_executable — which points to the Python interpreted from + # the base installation — isn't set (eg. when embedded), try to find + # it in 'home'. if not base_executable: # First try to resolve symlinked executables, since that may be # more accurate than assuming the executable in 'home'. @@ -400,6 +410,7 @@ def search_up(prefix, *landmarks, test=isfile): break break else: + # We didn't find a 'home' key in pyvenv.cfg (no break), reset venv_prefix. venv_prefix = None From 7b8bd3b2b81f4aca63c5b603b56998f6b3ee2611 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 15 Dec 2024 17:11:50 +0000 Subject: [PATCH 210/427] gh-119786: Fix miscellaneous typos in `InternalDocs/interpreter_definition.md` (#127957) --- Tools/cases_generator/interpreter_definition.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index d50c420307852f..7901f3d92e00bb 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -174,7 +174,7 @@ list of annotations and their meanings are as follows: * `override`. For external use by other interpreter definitions to override the current instruction definition. * `pure`. This instruction has no side effects. -* 'tierN'. This instruction only used by tier N interpreter. +* 'tierN'. This instruction is only used by the tier N interpreter. ### Special functions/macros @@ -393,7 +393,7 @@ which can be easily inserted. What is more complex is ensuring the correct stack and not generating excess pops and pushes. For example, in `CHECK_HAS_INSTANCE_VALUES`, `owner` occurs in the input, so it cannot be -redefined. Thus it doesn't need to be written and can be read without adjusting the stack pointer. +redefined. Thus, it doesn't need to be written and can be read without adjusting the stack pointer. The C code generated for `CHECK_HAS_INSTANCE_VALUES` would look something like: ```C @@ -404,7 +404,7 @@ The C code generated for `CHECK_HAS_INSTANCE_VALUES` would look something like: } ``` -When combining ops together to form instructions, temporary values should be used, +When combining ops to form instructions, temporary values should be used, rather than popping and pushing, such that `LOAD_ATTR_SLOT` would look something like: ```C From 46006a1b355f75d06c10e7b8086912c483b34487 Mon Sep 17 00:00:00 2001 From: Stephen Hansen Date: Sun, 15 Dec 2024 14:53:22 -0500 Subject: [PATCH 211/427] gh-127586: properly restore blocked signals in resource_tracker.py (GH-127587) * Correct pthread_sigmask in resource_tracker to restore old signals Using SIG_UNBLOCK to remove blocked "ignored signals" may accidentally cause side effects if the calling parent already had said signals blocked to begin with and did not intend to unblock them when creating a pool. Use SIG_SETMASK instead with the previous mask of blocked signals to restore the original blocked set. * Adding resource_tracker blocked signals test Co-authored-by: Peter Bierma Co-authored-by: Gregory P. Smith --- Lib/multiprocessing/resource_tracker.py | 7 ++++--- Lib/test/_test_multiprocessing.py | 15 +++++++++++++++ ...2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst | 3 +++ 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 20ddd9c50e3d88..90e036ae905afa 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -155,13 +155,14 @@ def ensure_running(self): # that can make the child die before it registers signal handlers # for SIGINT and SIGTERM. The mask is unregistered after spawning # the child. + prev_sigmask = None try: if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) pid = util.spawnv_passfds(exe, args, fds_to_pass) finally: - if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) + if prev_sigmask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) except: os.close(w) raise diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 80b08b8ac66899..01e92f0d8d6b30 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6044,6 +6044,21 @@ def test_resource_tracker_exit_code(self): self._test_resource_tracker_leak_resources( cleanup=cleanup, ) + @unittest.skipUnless(hasattr(signal, "pthread_sigmask"), "pthread_sigmask is not available") + def test_resource_tracker_blocked_signals(self): + # + # gh-127586: Check that resource_tracker does not override blocked signals of caller. + # + from multiprocessing.resource_tracker import ResourceTracker + signals = {signal.SIGTERM, signal.SIGINT, signal.SIGUSR1} + + for sig in signals: + signal.pthread_sigmask(signal.SIG_SETMASK, {sig}) + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker = ResourceTracker() + tracker.ensure_running() + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker._stop() class TestSimpleQueue(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst b/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst new file mode 100644 index 00000000000000..80217bd4a10503 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst @@ -0,0 +1,3 @@ +:class:`multiprocessing.pool.Pool` now properly restores blocked signal handlers +of the parent thread when creating processes via either *spawn* or +*forkserver*. From b74c8f58e875d909ce6b5b9dbcddd6d8331d2081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Sun, 15 Dec 2024 21:12:13 +0000 Subject: [PATCH 212/427] GH-126985: Don't override venv detection with PYTHONHOME (#127968) --- Lib/test/test_getpath.py | 31 +++++++++++++++++++++++++++++++ Modules/getpath.py | 10 +++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 7e5c4a3d14ddc5..f86df9d0d03485 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -832,6 +832,37 @@ def test_explicitly_set_stdlib_dir(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_PYTHONHOME_in_venv(self): + "Make sure prefix/exec_prefix still point to the venv if PYTHONHOME was used." + ns = MockPosixNamespace( + argv0="/venv/bin/python", + PREFIX="/usr", + ENV_PYTHONHOME="/pythonhome", + ) + # Setup venv + ns.add_known_xfile("/venv/bin/python") + ns.add_known_file("/venv/pyvenv.cfg", [ + r"home = /usr/bin" + ]) + # Seutup PYTHONHOME + ns.add_known_file("/pythonhome/lib/python9.8/os.py") + ns.add_known_dir("/pythonhome/lib/python9.8/lib-dynload") + + expected = dict( + executable="/venv/bin/python", + prefix="/venv", + exec_prefix="/venv", + base_prefix="/pythonhome", + base_exec_prefix="/pythonhome", + module_search_paths_set=1, + module_search_paths=[ + "/pythonhome/lib/python98.zip", + "/pythonhome/lib/python9.8", + "/pythonhome/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) # ****************************************************************************** diff --git a/Modules/getpath.py b/Modules/getpath.py index b14f985a0d5f97..c34101e720851d 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -344,9 +344,10 @@ def search_up(prefix, *landmarks, test=isfile): venv_prefix = None -# Calling Py_SetPythonHome(), Py_SetPath() or -# setting $PYTHONHOME will override venv detection. -if not home and not py_setpath: +# Calling Py_SetPath() will override venv detection. +# Calling Py_SetPythonHome() or setting $PYTHONHOME will override the 'home' key +# specified in pyvenv.cfg. +if not py_setpath: try: # prefix2 is just to avoid calculating dirname again later, # as the path in venv_prefix is the more common case. @@ -370,6 +371,9 @@ def search_up(prefix, *landmarks, test=isfile): for line in pyvenvcfg: key, had_equ, value = line.partition('=') if had_equ and key.strip().lower() == 'home': + # If PYTHONHOME was set, ignore 'home' from pyvenv.cfg. + if home: + break # Override executable_dir/real_executable_dir with the value from 'home'. # These values may be later used to calculate prefix/base_prefix, if a more # reliable source — like the runtime library (libpython) path — isn't available. From 47c5a0f307cff3ed477528536e8de095c0752efa Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 15 Dec 2024 23:17:01 +0000 Subject: [PATCH 213/427] gh-125588: Allow to regenerate the parser with Python < 3.12 (#127969) Signed-off-by: Pablo Galindo --- Tools/peg_generator/pegen/parser.py | 6 +++--- Tools/peg_generator/pegen/parser_generator.py | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index 692eb9ed2417d7..a987d30a9d6438 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -207,7 +207,7 @@ def string(self) -> Optional[tokenize.TokenInfo]: @memoize def fstring_start(self) -> Optional[tokenize.TokenInfo]: - FSTRING_START = getattr(token, "FSTRING_START") + FSTRING_START = getattr(token, "FSTRING_START", None) if not FSTRING_START: return None tok = self._tokenizer.peek() @@ -217,7 +217,7 @@ def fstring_start(self) -> Optional[tokenize.TokenInfo]: @memoize def fstring_middle(self) -> Optional[tokenize.TokenInfo]: - FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE") + FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE", None) if not FSTRING_MIDDLE: return None tok = self._tokenizer.peek() @@ -227,7 +227,7 @@ def fstring_middle(self) -> Optional[tokenize.TokenInfo]: @memoize def fstring_end(self) -> Optional[tokenize.TokenInfo]: - FSTRING_END = getattr(token, "FSTRING_END") + FSTRING_END = getattr(token, "FSTRING_END", None) if not FSTRING_END: return None tok = self._tokenizer.peek() diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index b42b12c8aa0dee..6ce0649aefe7ff 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -1,3 +1,4 @@ +import sys import ast import contextlib import re @@ -75,6 +76,11 @@ class RuleCheckingVisitor(GrammarVisitor): def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): self.rules = rules self.tokens = tokens + # If python < 3.12 add the virtual fstring tokens + if sys.version_info < (3, 12): + self.tokens.add("FSTRING_START") + self.tokens.add("FSTRING_END") + self.tokens.add("FSTRING_MIDDLE") def visit_NameLeaf(self, node: NameLeaf) -> None: if node.value not in self.rules and node.value not in self.tokens: From 0d8e7106c260e96c4604f501165bd106bff51f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:40:26 +0100 Subject: [PATCH 214/427] gh-111178: fix UBSan failures in `_elementtree.c` (#127982) --- Modules/_elementtree.c | 133 +++++++++++++++++++++++++---------------- 1 file changed, 80 insertions(+), 53 deletions(-) diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index e134e096e044b7..355f322d304c2f 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -184,7 +184,7 @@ elementtree_traverse(PyObject *m, visitproc visit, void *arg) static void elementtree_free(void *m) { - elementtree_clear((PyObject *)m); + (void)elementtree_clear((PyObject *)m); } /* helpers */ @@ -257,6 +257,7 @@ typedef struct { } ElementObject; +#define _Element_CAST(op) ((ElementObject *)(op)) #define Element_CheckExact(st, op) Py_IS_TYPE(op, (st)->Element_Type) #define Element_Check(st, op) PyObject_TypeCheck(op, (st)->Element_Type) @@ -648,8 +649,9 @@ subelement(PyObject *self, PyObject *args, PyObject *kwds) } static int -element_gc_traverse(ElementObject *self, visitproc visit, void *arg) +element_gc_traverse(PyObject *op, visitproc visit, void *arg) { + ElementObject *self = _Element_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->tag); Py_VISIT(JOIN_OBJ(self->text)); @@ -666,8 +668,9 @@ element_gc_traverse(ElementObject *self, visitproc visit, void *arg) } static int -element_gc_clear(ElementObject *self) +element_gc_clear(PyObject *op) { + ElementObject *self = _Element_CAST(op); Py_CLEAR(self->tag); _clear_joined_ptr(&self->text); _clear_joined_ptr(&self->tail); @@ -680,8 +683,9 @@ element_gc_clear(ElementObject *self) } static void -element_dealloc(ElementObject* self) +element_dealloc(PyObject *op) { + ElementObject *self = _Element_CAST(op); PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ @@ -689,13 +693,13 @@ element_dealloc(ElementObject* self) Py_TRASHCAN_BEGIN(self, element_dealloc) if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); + PyObject_ClearWeakRefs(op); /* element_gc_clear clears all references and deallocates extra */ - element_gc_clear(self); + (void)element_gc_clear(op); - tp->tp_free((PyObject *)self); + tp->tp_free(self); Py_DECREF(tp); Py_TRASHCAN_END } @@ -1478,9 +1482,9 @@ _elementtree_Element_itertext_impl(ElementObject *self, PyTypeObject *cls) static PyObject* -element_getitem(PyObject* self_, Py_ssize_t index) +element_getitem(PyObject *op, Py_ssize_t index) { - ElementObject* self = (ElementObject*) self_; + ElementObject *self = _Element_CAST(op); if (!self->extra || index < 0 || index >= self->extra->length) { PyErr_SetString( @@ -1494,9 +1498,9 @@ element_getitem(PyObject* self_, Py_ssize_t index) } static int -element_bool(PyObject* self_) +element_bool(PyObject *op) { - ElementObject* self = (ElementObject*) self_; + ElementObject *self = _Element_CAST(op); if (PyErr_WarnEx(PyExc_DeprecationWarning, "Testing an element's truth value will always return True " "in future versions. Use specific 'len(elem)' or " @@ -1583,8 +1587,9 @@ _elementtree_Element_keys_impl(ElementObject *self) } static Py_ssize_t -element_length(ElementObject* self) +element_length(PyObject *op) { + ElementObject *self = _Element_CAST(op); if (!self->extra) return 0; @@ -1675,10 +1680,10 @@ _elementtree_Element_remove_impl(ElementObject *self, PyObject *subelement) } static PyObject* -element_repr(ElementObject* self) +element_repr(PyObject *op) { int status; - + ElementObject *self = _Element_CAST(op); if (self->tag == NULL) return PyUnicode_FromFormat("", self); @@ -1728,9 +1733,9 @@ _elementtree_Element_set_impl(ElementObject *self, PyObject *key, } static int -element_setitem(PyObject* self_, Py_ssize_t index, PyObject* item) +element_setitem(PyObject *op, Py_ssize_t index, PyObject* item) { - ElementObject* self = (ElementObject*) self_; + ElementObject *self = _Element_CAST(op); Py_ssize_t i; PyObject* old; @@ -1762,10 +1767,10 @@ element_setitem(PyObject* self_, Py_ssize_t index, PyObject* item) return 0; } -static PyObject* -element_subscr(PyObject* self_, PyObject* item) +static PyObject * +element_subscr(PyObject *op, PyObject *item) { - ElementObject* self = (ElementObject*) self_; + ElementObject *self = _Element_CAST(op); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -1775,7 +1780,7 @@ element_subscr(PyObject* self_, PyObject* item) } if (i < 0 && self->extra) i += self->extra->length; - return element_getitem(self_, i); + return element_getitem(op, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen, i; @@ -1815,9 +1820,9 @@ element_subscr(PyObject* self_, PyObject* item) } static int -element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) +element_ass_subscr(PyObject *op, PyObject *item, PyObject *value) { - ElementObject* self = (ElementObject*) self_; + ElementObject *self = _Element_CAST(op); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -1827,7 +1832,7 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) } if (i < 0 && self->extra) i += self->extra->length; - return element_setitem(self_, i, value); + return element_setitem(op, i, value); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen, newlen, i; @@ -1998,30 +2003,34 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) } static PyObject* -element_tag_getter(ElementObject *self, void *closure) +element_tag_getter(PyObject *op, void *closure) { + ElementObject *self = _Element_CAST(op); PyObject *res = self->tag; return Py_NewRef(res); } static PyObject* -element_text_getter(ElementObject *self, void *closure) +element_text_getter(PyObject *op, void *closure) { + ElementObject *self = _Element_CAST(op); PyObject *res = element_get_text(self); return Py_XNewRef(res); } static PyObject* -element_tail_getter(ElementObject *self, void *closure) +element_tail_getter(PyObject *op, void *closure) { + ElementObject *self = _Element_CAST(op); PyObject *res = element_get_tail(self); return Py_XNewRef(res); } static PyObject* -element_attrib_getter(ElementObject *self, void *closure) +element_attrib_getter(PyObject *op, void *closure) { PyObject *res; + ElementObject *self = _Element_CAST(op); if (!self->extra) { if (create_extra(self, NULL) < 0) return NULL; @@ -2040,31 +2049,34 @@ element_attrib_getter(ElementObject *self, void *closure) } static int -element_tag_setter(ElementObject *self, PyObject *value, void *closure) +element_tag_setter(PyObject *op, PyObject *value, void *closure) { _VALIDATE_ATTR_VALUE(value); + ElementObject *self = _Element_CAST(op); Py_SETREF(self->tag, Py_NewRef(value)); return 0; } static int -element_text_setter(ElementObject *self, PyObject *value, void *closure) +element_text_setter(PyObject *op, PyObject *value, void *closure) { _VALIDATE_ATTR_VALUE(value); + ElementObject *self = _Element_CAST(op); _set_joined_ptr(&self->text, Py_NewRef(value)); return 0; } static int -element_tail_setter(ElementObject *self, PyObject *value, void *closure) +element_tail_setter(PyObject *op, PyObject *value, void *closure) { _VALIDATE_ATTR_VALUE(value); + ElementObject *self = _Element_CAST(op); _set_joined_ptr(&self->tail, Py_NewRef(value)); return 0; } static int -element_attrib_setter(ElementObject *self, PyObject *value, void *closure) +element_attrib_setter(PyObject *op, PyObject *value, void *closure) { _VALIDATE_ATTR_VALUE(value); if (!PyDict_Check(value)) { @@ -2073,6 +2085,7 @@ element_attrib_setter(ElementObject *self, PyObject *value, void *closure) Py_TYPE(value)->tp_name); return -1; } + ElementObject *self = _Element_CAST(op); if (!self->extra) { if (create_extra(self, NULL) < 0) return -1; @@ -2106,11 +2119,14 @@ typedef struct { int gettext; } ElementIterObject; +#define _ElementIter_CAST(op) ((ElementIterObject *)(op)) + static void -elementiter_dealloc(ElementIterObject *it) +elementiter_dealloc(PyObject *op) { - PyTypeObject *tp = Py_TYPE(it); + PyTypeObject *tp = Py_TYPE(op); + ElementIterObject *it = _ElementIter_CAST(op); Py_ssize_t i = it->parent_stack_used; it->parent_stack_used = 0; /* bpo-31095: UnTrack is needed before calling any callbacks */ @@ -2127,8 +2143,9 @@ elementiter_dealloc(ElementIterObject *it) } static int -elementiter_traverse(ElementIterObject *it, visitproc visit, void *arg) +elementiter_traverse(PyObject *op, visitproc visit, void *arg) { + ElementIterObject *it = _ElementIter_CAST(op); Py_ssize_t i = it->parent_stack_used; while (i--) Py_VISIT(it->parent_stack[i].parent); @@ -2162,8 +2179,9 @@ parent_stack_push_new(ElementIterObject *it, ElementObject *parent) } static PyObject * -elementiter_next(ElementIterObject *it) +elementiter_next(PyObject *op) { + ElementIterObject *it = _ElementIter_CAST(op); /* Sub-element iterator. * * A short note on gettext: this function serves both the iter() and @@ -2354,6 +2372,8 @@ typedef struct { elementtreestate *state; } TreeBuilderObject; + +#define _TreeBuilder_CAST(op) ((TreeBuilderObject *)(op)) #define TreeBuilder_CheckExact(st, op) Py_IS_TYPE((op), (st)->TreeBuilder_Type) /* -------------------------------------------------------------------- */ @@ -2444,8 +2464,9 @@ _elementtree_TreeBuilder___init___impl(TreeBuilderObject *self, } static int -treebuilder_gc_traverse(TreeBuilderObject *self, visitproc visit, void *arg) +treebuilder_gc_traverse(PyObject *op, visitproc visit, void *arg) { + TreeBuilderObject *self = _TreeBuilder_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->pi_event_obj); Py_VISIT(self->comment_event_obj); @@ -2467,8 +2488,9 @@ treebuilder_gc_traverse(TreeBuilderObject *self, visitproc visit, void *arg) } static int -treebuilder_gc_clear(TreeBuilderObject *self) +treebuilder_gc_clear(PyObject *op) { + TreeBuilderObject *self = _TreeBuilder_CAST(op); Py_CLEAR(self->pi_event_obj); Py_CLEAR(self->comment_event_obj); Py_CLEAR(self->end_ns_event_obj); @@ -2489,11 +2511,11 @@ treebuilder_gc_clear(TreeBuilderObject *self) } static void -treebuilder_dealloc(TreeBuilderObject *self) +treebuilder_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - treebuilder_gc_clear(self); + (void)treebuilder_gc_clear(self); tp->tp_free(self); Py_DECREF(tp); } @@ -3061,6 +3083,9 @@ typedef struct { PyObject *elementtree_module; } XMLParserObject; + +#define _XMLParser_CAST(op) ((XMLParserObject *)(op)) + /* helpers */ LOCAL(PyObject*) @@ -3751,8 +3776,9 @@ _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *target, } static int -xmlparser_gc_traverse(XMLParserObject *self, visitproc visit, void *arg) +xmlparser_gc_traverse(PyObject *op, visitproc visit, void *arg) { + XMLParserObject *self = _XMLParser_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->handle_close); Py_VISIT(self->handle_pi); @@ -3772,8 +3798,9 @@ xmlparser_gc_traverse(XMLParserObject *self, visitproc visit, void *arg) } static int -xmlparser_gc_clear(XMLParserObject *self) +xmlparser_gc_clear(PyObject *op) { + XMLParserObject *self = _XMLParser_CAST(op); elementtreestate *st = self->state; if (self->parser != NULL) { XML_Parser parser = self->parser; @@ -3800,11 +3827,11 @@ xmlparser_gc_clear(XMLParserObject *self) } static void -xmlparser_dealloc(XMLParserObject* self) +xmlparser_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - xmlparser_gc_clear(self); + (void)xmlparser_gc_clear(self); tp->tp_free(self); Py_DECREF(tp); } @@ -4172,7 +4199,7 @@ static PyMemberDef xmlparser_members[] = { }; static PyObject* -xmlparser_version_getter(XMLParserObject *self, void *closure) +xmlparser_version_getter(PyObject *op, void *closure) { return PyUnicode_FromFormat( "Expat %d.%d.%d", XML_MAJOR_VERSION, @@ -4180,7 +4207,7 @@ xmlparser_version_getter(XMLParserObject *self, void *closure) } static PyGetSetDef xmlparser_getsetlist[] = { - {"version", (getter)xmlparser_version_getter, NULL, NULL}, + {"version", xmlparser_version_getter, NULL, NULL}, {NULL}, }; @@ -4229,20 +4256,20 @@ static struct PyMemberDef element_members[] = { static PyGetSetDef element_getsetlist[] = { {"tag", - (getter)element_tag_getter, - (setter)element_tag_setter, + element_tag_getter, + element_tag_setter, "A string identifying what kind of data this element represents"}, {"text", - (getter)element_text_getter, - (setter)element_text_setter, + element_text_getter, + element_text_setter, "A string of text directly after the start tag, or None"}, {"tail", - (getter)element_tail_getter, - (setter)element_tail_setter, + element_tail_getter, + element_tail_setter, "A string of text directly after the end tag, or None"}, {"attrib", - (getter)element_attrib_getter, - (setter)element_attrib_setter, + element_attrib_getter, + element_attrib_setter, "A dictionary containing the element's attributes"}, {NULL}, }; From 52d552cda7614c7aa9f08b680089c630587e747f Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Mon, 16 Dec 2024 22:56:04 +0900 Subject: [PATCH 215/427] gh-127896: Add missing documentation of `PySequence_In` (GH-127979) Co-authored-by: Sergey B Kirpichev --- Doc/c-api/sequence.rst | 9 +++++++++ Doc/whatsnew/3.14.rst | 4 ++++ .../C_API/2024-12-16-07-12-15.gh-issue-127896.HmI9pk.rst | 2 ++ 3 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/C_API/2024-12-16-07-12-15.gh-issue-127896.HmI9pk.rst diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index ce28839f5ba739..df5bf6b64a93a0 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -105,6 +105,15 @@ Sequence Protocol equivalent to the Python expression ``value in o``. +.. c:function:: int PySequence_In(PyObject *o, PyObject *value) + + Alias for :c:func:`PySequence_Contains`. + + .. deprecated:: 3.14 + The function is :term:`soft deprecated` and should no longer be used to + write new code. + + .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) Return the first index *i* for which ``o[i] == value``. On error, return diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 095949242c09d9..d13cd2d5173a04 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1073,6 +1073,10 @@ Deprecated :c:macro:`!isfinite` available from :file:`math.h` since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) +* The previously undocumented function :c:func:`PySequence_In` is :term:`soft deprecated`. + Use :c:func:`PySequence_Contains` instead. + (Contributed by Yuki Kobayashi in :gh:`127896`.) + .. Add C API deprecations above alphabetically, not here at the end. .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst diff --git a/Misc/NEWS.d/next/C_API/2024-12-16-07-12-15.gh-issue-127896.HmI9pk.rst b/Misc/NEWS.d/next/C_API/2024-12-16-07-12-15.gh-issue-127896.HmI9pk.rst new file mode 100644 index 00000000000000..82b4f563591fe1 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-16-07-12-15.gh-issue-127896.HmI9pk.rst @@ -0,0 +1,2 @@ +The previously undocumented function :c:func:`PySequence_In` is :term:`soft deprecated`. +Use :c:func:`PySequence_Contains` instead. From 081673801e3d47d931d2e2b6a4a1515e1207d938 Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Mon, 16 Dec 2024 17:57:18 +0100 Subject: [PATCH 216/427] gh-127864: Fix compiler warning (-Wstringop-truncation) (GH-127878) --- Python/import.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/import.c b/Python/import.c index f3511aaf7b8010..a9282dde633959 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1176,9 +1176,10 @@ hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) return NULL; } - strncpy(key, str1_data, str1_len); + memcpy(key, str1_data, str1_len); key[str1_len] = sep; - strncpy(key + str1_len + 1, str2_data, str2_len + 1); + memcpy(key + str1_len + 1, str2_data, str2_len); + key[size - 1] = '\0'; assert(strlen(key) == size - 1); return key; } From e4981e33b82ac14cca0f2d9b95257301fa201810 Mon Sep 17 00:00:00 2001 From: Gugubo <29143981+Gugubo@users.noreply.github.com> Date: Mon, 16 Dec 2024 18:08:25 +0100 Subject: [PATCH 217/427] Fix typo in itertools docs (gh-127995) --- Doc/library/itertools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index d1fb952e36f686..eb61453718bd3c 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -681,7 +681,7 @@ loops that truncate the stream. consumed from the input iterator and there is no way to access it. This could be an issue if an application wants to further consume the input iterator after *takewhile* has been run to exhaustion. To work - around this problem, consider using `more-iterools before_and_after() + around this problem, consider using `more-itertools before_and_after() `_ instead. From 1d276ec6f8403590a6a1a18c560ce75b9221572b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 16 Dec 2024 20:18:02 +0200 Subject: [PATCH 218/427] Revert "gh-127586: properly restore blocked signals in resource_tracker.py (GH-127587)" (#127983) This reverts commit 46006a1b355f75d06c10e7b8086912c483b34487. --- Lib/multiprocessing/resource_tracker.py | 7 +++---- Lib/test/_test_multiprocessing.py | 15 --------------- ...2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst | 3 --- 3 files changed, 3 insertions(+), 22 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 90e036ae905afa..20ddd9c50e3d88 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -155,14 +155,13 @@ def ensure_running(self): # that can make the child die before it registers signal handlers # for SIGINT and SIGTERM. The mask is unregistered after spawning # the child. - prev_sigmask = None try: if _HAVE_SIGMASK: - prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) pid = util.spawnv_passfds(exe, args, fds_to_pass) finally: - if prev_sigmask is not None: - signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) + if _HAVE_SIGMASK: + signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) except: os.close(w) raise diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 01e92f0d8d6b30..80b08b8ac66899 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6044,21 +6044,6 @@ def test_resource_tracker_exit_code(self): self._test_resource_tracker_leak_resources( cleanup=cleanup, ) - @unittest.skipUnless(hasattr(signal, "pthread_sigmask"), "pthread_sigmask is not available") - def test_resource_tracker_blocked_signals(self): - # - # gh-127586: Check that resource_tracker does not override blocked signals of caller. - # - from multiprocessing.resource_tracker import ResourceTracker - signals = {signal.SIGTERM, signal.SIGINT, signal.SIGUSR1} - - for sig in signals: - signal.pthread_sigmask(signal.SIG_SETMASK, {sig}) - self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) - tracker = ResourceTracker() - tracker.ensure_running() - self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) - tracker._stop() class TestSimpleQueue(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst b/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst deleted file mode 100644 index 80217bd4a10503..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`multiprocessing.pool.Pool` now properly restores blocked signal handlers -of the parent thread when creating processes via either *spawn* or -*forkserver*. From 4937ba54c0ff7cc4a83d7345d398b804365af2d6 Mon Sep 17 00:00:00 2001 From: Edward Xu Date: Tue, 17 Dec 2024 03:12:19 +0800 Subject: [PATCH 219/427] gh-127085: fix some data races in memoryview in free-threading (#127412) --- Lib/test/test_memoryview.py | 26 ++++++++++++++- ...-11-30-23-35-45.gh-issue-127085.KLKylb.rst | 1 + Objects/memoryobject.c | 33 +++++++++++++++---- 3 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 96320ebef6467a..856e17918048e4 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -16,7 +16,8 @@ import struct from itertools import product -from test.support import import_helper +from test import support +from test.support import import_helper, threading_helper class MyObject: @@ -733,5 +734,28 @@ def test_picklebuffer_reference_loop(self): self.assertIsNone(wr()) +@threading_helper.requires_working_threading() +@support.requires_resource("cpu") +class RacingTest(unittest.TestCase): + def test_racing_getbuf_and_releasebuf(self): + """Repeatly access the memoryview for racing.""" + from multiprocessing.managers import SharedMemoryManager + from threading import Thread + + n = 100 + with SharedMemoryManager() as smm: + obj = smm.ShareableList(range(100)) + threads = [] + for _ in range(n): + # Issue gh-127085, the `ShareableList.count` is just a convenient way to mess the `exports` + # counter of `memoryview`, this issue has no direct relation with `ShareableList`. + threads.append(Thread(target=obj.count, args=(1,))) + + with threading_helper.start_threads(threads): + pass + + del obj + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst new file mode 100644 index 00000000000000..a59b9502eaa33e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst @@ -0,0 +1 @@ +Fix race when exporting a buffer from a :class:`memoryview` object on the :term:`free-threaded ` build. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index afe2dcb127adb4..ea4d24dc690768 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1086,6 +1086,16 @@ PyBuffer_ToContiguous(void *buf, const Py_buffer *src, Py_ssize_t len, char orde return ret; } +static inline Py_ssize_t +get_exports(PyMemoryViewObject *buf) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&buf->exports); +#else + return buf->exports; +#endif +} + /****************************************************************************/ /* Release/GC management */ @@ -1098,7 +1108,7 @@ PyBuffer_ToContiguous(void *buf, const Py_buffer *src, Py_ssize_t len, char orde static void _memory_release(PyMemoryViewObject *self) { - assert(self->exports == 0); + assert(get_exports(self) == 0); if (self->flags & _Py_MEMORYVIEW_RELEASED) return; @@ -1119,15 +1129,16 @@ static PyObject * memoryview_release_impl(PyMemoryViewObject *self) /*[clinic end generated code: output=d0b7e3ba95b7fcb9 input=bc71d1d51f4a52f0]*/ { - if (self->exports == 0) { + Py_ssize_t exports = get_exports(self); + if (exports == 0) { _memory_release(self); Py_RETURN_NONE; } - if (self->exports > 0) { + if (exports > 0) { PyErr_Format(PyExc_BufferError, - "memoryview has %zd exported buffer%s", self->exports, - self->exports==1 ? "" : "s"); + "memoryview has %zd exported buffer%s", exports, + exports==1 ? "" : "s"); return NULL; } @@ -1140,7 +1151,7 @@ static void memory_dealloc(PyObject *_self) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - assert(self->exports == 0); + assert(get_exports(self) == 0); _PyObject_GC_UNTRACK(self); _memory_release(self); Py_CLEAR(self->mbuf); @@ -1161,7 +1172,7 @@ static int memory_clear(PyObject *_self) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - if (self->exports == 0) { + if (get_exports(self) == 0) { _memory_release(self); Py_CLEAR(self->mbuf); } @@ -1589,7 +1600,11 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); +#ifdef Py_GIL_DISABLED + _Py_atomic_add_ssize(&self->exports, 1); +#else self->exports++; +#endif return 0; } @@ -1598,7 +1613,11 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; +#ifdef Py_GIL_DISABLED + _Py_atomic_add_ssize(&self->exports, -1); +#else self->exports--; +#endif return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } From 3b766824fe59030100964752be0556084d4461af Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 16 Dec 2024 14:31:44 -0500 Subject: [PATCH 220/427] gh-126907: make `atexit` thread safe in free-threading (#127935) Co-authored-by: Victor Stinner Co-authored-by: Kumar Aditya --- Include/internal/pycore_atexit.h | 24 +-- Lib/test/test_atexit.py | 35 +++- ...-12-13-22-20-54.gh-issue-126907.fWRL_R.rst | 2 + Modules/atexitmodule.c | 153 +++++++++--------- 4 files changed, 126 insertions(+), 88 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-13-22-20-54.gh-issue-126907.fWRL_R.rst diff --git a/Include/internal/pycore_atexit.h b/Include/internal/pycore_atexit.h index cde5b530baf00c..db1e5568e09413 100644 --- a/Include/internal/pycore_atexit.h +++ b/Include/internal/pycore_atexit.h @@ -36,23 +36,29 @@ typedef struct atexit_callback { struct atexit_callback *next; } atexit_callback; -typedef struct { - PyObject *func; - PyObject *args; - PyObject *kwargs; -} atexit_py_callback; - struct atexit_state { +#ifdef Py_GIL_DISABLED + PyMutex ll_callbacks_lock; +#endif atexit_callback *ll_callbacks; // XXX The rest of the state could be moved to the atexit module state // and a low-level callback added for it during module exec. // For the moment we leave it here. - atexit_py_callback **callbacks; - int ncallbacks; - int callback_len; + + // List containing tuples with callback information. + // e.g. [(func, args, kwargs), ...] + PyObject *callbacks; }; +#ifdef Py_GIL_DISABLED +# define _PyAtExit_LockCallbacks(state) PyMutex_Lock(&state->ll_callbacks_lock); +# define _PyAtExit_UnlockCallbacks(state) PyMutex_Unlock(&state->ll_callbacks_lock); +#else +# define _PyAtExit_LockCallbacks(state) +# define _PyAtExit_UnlockCallbacks(state) +#endif + // Export for '_interpchannels' shared extension PyAPI_FUNC(int) _Py_AtExit( PyInterpreterState *interp, diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 913b7556be83af..eb01da6e88a8bc 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -4,7 +4,7 @@ import unittest from test import support from test.support import script_helper - +from test.support import threading_helper class GeneralTest(unittest.TestCase): def test_general(self): @@ -46,6 +46,39 @@ def test_atexit_instances(self): self.assertEqual(res.out.decode().splitlines(), ["atexit2", "atexit1"]) self.assertFalse(res.err) + @threading_helper.requires_working_threading() + @support.requires_resource("cpu") + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful without the GIL") + def test_atexit_thread_safety(self): + # GH-126907: atexit was not thread safe on the free-threaded build + source = """ + from threading import Thread + + def dummy(): + pass + + + def thready(): + for _ in range(100): + atexit.register(dummy) + atexit._clear() + atexit.register(dummy) + atexit.unregister(dummy) + atexit._run_exitfuncs() + + + threads = [Thread(target=thready) for _ in range(10)] + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + """ + + # atexit._clear() has some evil side effects, and we don't + # want them to affect the rest of the tests. + script_helper.assert_python_ok("-c", textwrap.dedent(source)) + @support.cpython_only class SubinterpreterTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-12-13-22-20-54.gh-issue-126907.fWRL_R.rst b/Misc/NEWS.d/next/Library/2024-12-13-22-20-54.gh-issue-126907.fWRL_R.rst new file mode 100644 index 00000000000000..d33d2aa23b21b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-13-22-20-54.gh-issue-126907.fWRL_R.rst @@ -0,0 +1,2 @@ +Fix crash when using :mod:`atexit` concurrently on the :term:`free-threaded +` build. diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index c009235b7a36c2..1b89b32ba907d7 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -41,6 +41,7 @@ PyUnstable_AtExit(PyInterpreterState *interp, callback->next = NULL; struct atexit_state *state = &interp->atexit; + _PyAtExit_LockCallbacks(state); atexit_callback *top = state->ll_callbacks; if (top == NULL) { state->ll_callbacks = callback; @@ -49,36 +50,16 @@ PyUnstable_AtExit(PyInterpreterState *interp, callback->next = top; state->ll_callbacks = callback; } + _PyAtExit_UnlockCallbacks(state); return 0; } -static void -atexit_delete_cb(struct atexit_state *state, int i) -{ - atexit_py_callback *cb = state->callbacks[i]; - state->callbacks[i] = NULL; - - Py_DECREF(cb->func); - Py_DECREF(cb->args); - Py_XDECREF(cb->kwargs); - PyMem_Free(cb); -} - - /* Clear all callbacks without calling them */ static void atexit_cleanup(struct atexit_state *state) { - atexit_py_callback *cb; - for (int i = 0; i < state->ncallbacks; i++) { - cb = state->callbacks[i]; - if (cb == NULL) - continue; - - atexit_delete_cb(state, i); - } - state->ncallbacks = 0; + PyList_Clear(state->callbacks); } @@ -89,23 +70,21 @@ _PyAtExit_Init(PyInterpreterState *interp) // _PyAtExit_Init() must only be called once assert(state->callbacks == NULL); - state->callback_len = 32; - state->ncallbacks = 0; - state->callbacks = PyMem_New(atexit_py_callback*, state->callback_len); + state->callbacks = PyList_New(0); if (state->callbacks == NULL) { return _PyStatus_NO_MEMORY(); } return _PyStatus_OK(); } - void _PyAtExit_Fini(PyInterpreterState *interp) { + // In theory, there shouldn't be any threads left by now, so we + // won't lock this. struct atexit_state *state = &interp->atexit; atexit_cleanup(state); - PyMem_Free(state->callbacks); - state->callbacks = NULL; + Py_CLEAR(state->callbacks); atexit_callback *next = state->ll_callbacks; state->ll_callbacks = NULL; @@ -120,35 +99,44 @@ _PyAtExit_Fini(PyInterpreterState *interp) } } - static void atexit_callfuncs(struct atexit_state *state) { assert(!PyErr_Occurred()); + assert(state->callbacks != NULL); + assert(PyList_CheckExact(state->callbacks)); - if (state->ncallbacks == 0) { + // Create a copy of the list for thread safety + PyObject *copy = PyList_GetSlice(state->callbacks, 0, PyList_GET_SIZE(state->callbacks)); + if (copy == NULL) + { + PyErr_WriteUnraisable(NULL); return; } - for (int i = state->ncallbacks - 1; i >= 0; i--) { - atexit_py_callback *cb = state->callbacks[i]; - if (cb == NULL) { - continue; - } + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(copy); ++i) { + // We don't have to worry about evil borrowed references, because + // no other threads can access this list. + PyObject *tuple = PyList_GET_ITEM(copy, i); + assert(PyTuple_CheckExact(tuple)); + + PyObject *func = PyTuple_GET_ITEM(tuple, 0); + PyObject *args = PyTuple_GET_ITEM(tuple, 1); + PyObject *kwargs = PyTuple_GET_ITEM(tuple, 2); - // bpo-46025: Increment the refcount of cb->func as the call itself may unregister it - PyObject* the_func = Py_NewRef(cb->func); - PyObject *res = PyObject_Call(cb->func, cb->args, cb->kwargs); + PyObject *res = PyObject_Call(func, + args, + kwargs == Py_None ? NULL : kwargs); if (res == NULL) { PyErr_FormatUnraisable( - "Exception ignored in atexit callback %R", the_func); + "Exception ignored in atexit callback %R", func); } else { Py_DECREF(res); } - Py_DECREF(the_func); } + Py_DECREF(copy); atexit_cleanup(state); assert(!PyErr_Occurred()); @@ -194,33 +182,27 @@ atexit_register(PyObject *module, PyObject *args, PyObject *kwargs) "the first argument must be callable"); return NULL; } + PyObject *func_args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); + PyObject *func_kwargs = kwargs; - struct atexit_state *state = get_atexit_state(); - if (state->ncallbacks >= state->callback_len) { - atexit_py_callback **r; - state->callback_len += 16; - size_t size = sizeof(atexit_py_callback*) * (size_t)state->callback_len; - r = (atexit_py_callback**)PyMem_Realloc(state->callbacks, size); - if (r == NULL) { - return PyErr_NoMemory(); - } - state->callbacks = r; + if (func_kwargs == NULL) + { + func_kwargs = Py_None; } - - atexit_py_callback *callback = PyMem_Malloc(sizeof(atexit_py_callback)); - if (callback == NULL) { - return PyErr_NoMemory(); + PyObject *callback = PyTuple_Pack(3, func, func_args, func_kwargs); + if (callback == NULL) + { + return NULL; } - callback->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); - if (callback->args == NULL) { - PyMem_Free(callback); + struct atexit_state *state = get_atexit_state(); + // atexit callbacks go in a LIFO order + if (PyList_Insert(state->callbacks, 0, callback) < 0) + { + Py_DECREF(callback); return NULL; } - callback->func = Py_NewRef(func); - callback->kwargs = Py_XNewRef(kwargs); - - state->callbacks[state->ncallbacks++] = callback; + Py_DECREF(callback); return Py_NewRef(func); } @@ -264,7 +246,33 @@ static PyObject * atexit_ncallbacks(PyObject *module, PyObject *unused) { struct atexit_state *state = get_atexit_state(); - return PyLong_FromSsize_t(state->ncallbacks); + assert(state->callbacks != NULL); + assert(PyList_CheckExact(state->callbacks)); + return PyLong_FromSsize_t(PyList_GET_SIZE(state->callbacks)); +} + +static int +atexit_unregister_locked(PyObject *callbacks, PyObject *func) +{ + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(callbacks); ++i) { + PyObject *tuple = PyList_GET_ITEM(callbacks, i); + assert(PyTuple_CheckExact(tuple)); + PyObject *to_compare = PyTuple_GET_ITEM(tuple, 0); + int cmp = PyObject_RichCompareBool(func, to_compare, Py_EQ); + if (cmp < 0) + { + return -1; + } + if (cmp == 1) { + // We found a callback! + if (PyList_SetSlice(callbacks, i, i + 1, NULL) < 0) { + return -1; + } + --i; + } + } + + return 0; } PyDoc_STRVAR(atexit_unregister__doc__, @@ -280,22 +288,11 @@ static PyObject * atexit_unregister(PyObject *module, PyObject *func) { struct atexit_state *state = get_atexit_state(); - for (int i = 0; i < state->ncallbacks; i++) - { - atexit_py_callback *cb = state->callbacks[i]; - if (cb == NULL) { - continue; - } - - int eq = PyObject_RichCompareBool(cb->func, func, Py_EQ); - if (eq < 0) { - return NULL; - } - if (eq) { - atexit_delete_cb(state, i); - } - } - Py_RETURN_NONE; + int result; + Py_BEGIN_CRITICAL_SECTION(state->callbacks); + result = atexit_unregister_locked(state->callbacks, func); + Py_END_CRITICAL_SECTION(); + return result < 0 ? NULL : Py_None; } From cfeaa992ba9bad9be2687afcafd85156703d74e8 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 16 Dec 2024 22:59:36 +0200 Subject: [PATCH 221/427] Free arena on _PyCompile_AstOptimize failure in Py_CompileStringObject (GH-127910) After commit 10a91d7e9 introduced arena cleanup, commit 2dfbd4f36 removed the free call when _PyCompile_AstOptimize fails. --- Python/pythonrun.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 8b57018321c070..31e065ff00d59a 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1486,6 +1486,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { if ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_OPTIMIZED_AST) { if (_PyCompile_AstOptimize(mod, filename, flags, optimize, arena) < 0) { + _PyArena_Free(arena); return NULL; } } From 1183e4ce2f7c07aeeff7c757ec749ef5af9d4415 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 17 Dec 2024 08:48:23 +0100 Subject: [PATCH 222/427] gh-127146: Emscripten clean up test suite (#127984) Removed test skips that are no longer required as a result of Emscripten updates. --- Lib/test/libregrtest/setup.py | 4 +-- Lib/test/libregrtest/utils.py | 11 ------- Lib/test/test_fileio.py | 4 +-- Lib/test/test_logging.py | 3 -- Lib/test/test_os.py | 44 ++++++++----------------- Lib/test/test_pathlib/test_pathlib.py | 7 +--- Lib/test/test_pydoc/test_pydoc.py | 1 - Lib/test/test_strptime.py | 6 ---- Lib/test/test_support.py | 1 - Lib/test/test_tempfile.py | 10 ------ Lib/test/test_time.py | 6 ---- Lib/test/test_unicode_file_functions.py | 4 +-- 12 files changed, 19 insertions(+), 82 deletions(-) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index ba57f06b4841d4..c0346aa934d394 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -11,7 +11,7 @@ from .filter import set_match_tests from .runtests import RunTests from .utils import ( - setup_unraisable_hook, setup_threading_excepthook, fix_umask, + setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -26,8 +26,6 @@ def setup_test_dir(testdir: str | None) -> None: def setup_process() -> None: - fix_umask() - assert sys.__stderr__ is not None, "sys.__stderr__ is None" try: stderr_fd = sys.__stderr__.fileno() diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 1ecfc61af9e39d..3eff9e753b6d84 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -478,17 +478,6 @@ def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: return os.path.abspath(tmp_dir) -def fix_umask() -> None: - if support.is_emscripten: - # Emscripten has default umask 0o777, which breaks some tests. - # see https://github.com/emscripten-core/emscripten/issues/17269 - old_mask = os.umask(0) - if old_mask == 0o777: - os.umask(0o027) - else: - os.umask(old_mask) - - def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath: # Define a writable temp dir that will be used as cwd while running # the tests. The name of the dir includes the pid to allow parallel diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index e681417e15d34b..5a0f033ebb82d2 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,7 +10,7 @@ from functools import wraps from test.support import ( - cpython_only, swap_attr, gc_collect, is_emscripten, is_wasi, + cpython_only, swap_attr, gc_collect, is_wasi, infinite_recursion, strace_helper ) from test.support.os_helper import ( @@ -531,7 +531,7 @@ def testAbles(self): self.assertEqual(f.isatty(), False) f.close() - if sys.platform != "win32" and not is_emscripten: + if sys.platform != "win32": try: f = self.FileIO("/dev/tty", "a") except OSError: diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index e72f222e1c7eeb..671b4c57a809aa 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -680,9 +680,6 @@ def test_pathlike_objects(self): os.unlink(fn) @unittest.skipIf(os.name == 'nt', 'WatchedFileHandler not appropriate for Windows.') - @unittest.skipIf( - support.is_emscripten, "Emscripten cannot fstat unlinked files." - ) @threading_helper.requires_working_threading() @support.requires_resource('walltime') def test_race(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 8aac92934f6ac0..b0e686cb754b93 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -188,9 +188,6 @@ def test_access(self): os.close(f) self.assertTrue(os.access(os_helper.TESTFN, os.W_OK)) - @unittest.skipIf( - support.is_emscripten, "Test is unstable under Emscripten." - ) @unittest.skipIf( support.is_wasi, "WASI does not support dup." ) @@ -1428,9 +1425,7 @@ def setUp(self): else: self.sub2_tree = (sub2_path, ["SUB21"], ["tmp3"]) - if not support.is_emscripten: - # Emscripten fails with inaccessible directory - os.chmod(sub21_path, 0) + os.chmod(sub21_path, 0) try: os.listdir(sub21_path) except PermissionError: @@ -1726,9 +1721,6 @@ def test_yields_correct_dir_fd(self): # check that listdir() returns consistent information self.assertEqual(set(os.listdir(rootfd)), set(dirs) | set(files)) - @unittest.skipIf( - support.is_emscripten, "Cannot dup stdout on Emscripten" - ) @unittest.skipIf( support.is_android, "dup return value is unpredictable on Android" ) @@ -1745,9 +1737,6 @@ def test_fd_leak(self): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) - @unittest.skipIf( - support.is_emscripten, "Cannot dup stdout on Emscripten" - ) @unittest.skipIf( support.is_android, "dup return value is unpredictable on Android" ) @@ -1816,8 +1805,8 @@ def test_makedir(self): os.makedirs(path) @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "Emscripten's/WASI's umask is a stub." + support.is_wasi, + "WASI's umask is a stub." ) def test_mode(self): with os_helper.temp_umask(0o002): @@ -1832,8 +1821,8 @@ def test_mode(self): self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "Emscripten's/WASI's umask is a stub." + support.is_wasi, + "WASI's umask is a stub." ) def test_exist_ok_existing_directory(self): path = os.path.join(os_helper.TESTFN, 'dir1') @@ -1850,8 +1839,8 @@ def test_exist_ok_existing_directory(self): os.makedirs(os.path.abspath('/'), exist_ok=True) @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "Emscripten's/WASI's umask is a stub." + support.is_wasi, + "WASI's umask is a stub." ) def test_exist_ok_s_isgid_directory(self): path = os.path.join(os_helper.TESTFN, 'dir1') @@ -2429,10 +2418,6 @@ def test_dup2(self): self.check(os.dup2, 20) @unittest.skipUnless(hasattr(os, 'dup2'), 'test needs os.dup2()') - @unittest.skipIf( - support.is_emscripten, - "dup2() with negative fds is broken on Emscripten (see gh-102179)" - ) def test_dup2_negative_fd(self): valid_fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, valid_fd) @@ -2457,14 +2442,14 @@ def test_fchown(self): self.check(os.fchown, -1, -1) @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "musl libc issue on Emscripten/WASI, bpo-46390" - ) def test_fpathconf(self): self.assertIn("PC_NAME_MAX", os.pathconf_names) - self.check(os.pathconf, "PC_NAME_MAX") - self.check(os.fpathconf, "PC_NAME_MAX") + if not (support.is_emscripten or support.is_wasi): + # musl libc pathconf ignores the file descriptor and always returns + # a constant, so the assertion that it should notice a bad file + # descriptor and return EBADF fails. + self.check(os.pathconf, "PC_NAME_MAX") + self.check(os.fpathconf, "PC_NAME_MAX") self.check_bool(os.pathconf, "PC_NAME_MAX") self.check_bool(os.fpathconf, "PC_NAME_MAX") @@ -3395,9 +3380,6 @@ def test_bad_fd(self): @unittest.skipUnless(os.isatty(0) and not win32_is_iot() and (sys.platform.startswith('win') or (hasattr(locale, 'nl_langinfo') and hasattr(locale, 'CODESET'))), 'test requires a tty and either Windows or nl_langinfo(CODESET)') - @unittest.skipIf( - support.is_emscripten, "Cannot get encoding of stdin on Emscripten" - ) def test_device_encoding(self): encoding = os.device_encoding(0) self.assertIsNotNone(encoding) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 68bff2cf0d511e..ac3a3b4f15c10e 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1673,7 +1673,6 @@ def test_mkdir_exist_ok_with_parent(self): self.assertTrue(p.exists()) self.assertEqual(p.stat().st_ctime, st_ctime_first) - @unittest.skipIf(is_emscripten, "FS root cannot be modified on Emscripten.") def test_mkdir_exist_ok_root(self): # Issue #25803: A drive root could raise PermissionError on Windows. self.cls('/').resolve().mkdir(exist_ok=True) @@ -2039,7 +2038,6 @@ def test_rglob_pathlike(self): self.assertEqual(expect, set(p.rglob(FakePath(pattern)))) @needs_symlinks - @unittest.skipIf(is_emscripten, "Hangs") def test_glob_recurse_symlinks_common(self): def _check(path, glob, expected): actual = {path for path in path.glob(glob, recurse_symlinks=True) @@ -2077,7 +2075,6 @@ def _check(path, glob, expected): _check(p, "*/dirD/**/", ["dirC/dirD/"]) @needs_symlinks - @unittest.skipIf(is_emscripten, "Hangs") def test_rglob_recurse_symlinks_common(self): def _check(path, glob, expected): actual = {path for path in path.rglob(glob, recurse_symlinks=True) @@ -2484,9 +2481,7 @@ def setUp(self): os.symlink(tmp5_path, broken_link3_path) self.sub2_tree[2].append('broken_link3') self.sub2_tree[2].sort() - if not is_emscripten: - # Emscripten fails with inaccessible directories. - os.chmod(sub21_path, 0) + os.chmod(sub21_path, 0) try: os.listdir(sub21_path) except PermissionError: diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 2a4d3ab73db608..3283fde9e12a8a 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -1224,7 +1224,6 @@ def test_apropos_with_unreadable_dir(self): self.assertEqual(err.getvalue(), '') @os_helper.skip_unless_working_chmod - @unittest.skipIf(is_emscripten, "cannot remove x bit") def test_apropos_empty_doc(self): pkgdir = os.path.join(TESTFN, 'walkpkg') os.mkdir(pkgdir) diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 9f5cfca9c7f124..0d30a63ab0c140 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -79,9 +79,6 @@ def test_am_pm(self): self.assertEqual(self.LT_ins.am_pm[position], strftime_output, "AM/PM representation in the wrong position within the tuple") - @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" - ) def test_timezone(self): # Make sure timezone is correct timezone = time.strftime("%Z", self.time_tuple).lower() @@ -431,9 +428,6 @@ def test_bad_offset(self): self.assertEqual("Inconsistent use of : in -01:3030", str(err.exception)) @skip_if_buggy_ucrt_strfptime - @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" - ) def test_timezone(self): # Test timezone directives. # When gmtime() is used with %Z, entire result of strftime() is empty. diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 635ae03a404988..d900db546ada8d 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -549,7 +549,6 @@ def test_optim_args_from_interpreter_flags(self): self.check_options(opts, 'optim_args_from_interpreter_flags') @unittest.skipIf(support.is_apple_mobile, "Unstable on Apple Mobile") - @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): # We cannot test the absolute value of fd_count(): on old Linux kernel diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index a5e182cef23dc5..57e9bd20c77ee1 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -328,10 +328,6 @@ def _mock_candidate_names(*names): class TestBadTempdir: - - @unittest.skipIf( - support.is_emscripten, "Emscripten cannot remove write bits." - ) def test_read_only_directory(self): with _inside_empty_temp_dir(): oldmode = mode = os.stat(tempfile.tempdir).st_mode @@ -1240,9 +1236,6 @@ def test_del_unrolled_file(self): with self.assertWarns(ResourceWarning): f.__del__() - @unittest.skipIf( - support.is_emscripten, "Emscripten cannot fstat renamed files." - ) def test_del_rolled_file(self): # The rolled file should be deleted when the SpooledTemporaryFile # object is deleted. This should raise a ResourceWarning since the file @@ -1468,9 +1461,6 @@ def use_closed(): pass self.assertRaises(ValueError, use_closed) - @unittest.skipIf( - support.is_emscripten, "Emscripten cannot fstat renamed files." - ) def test_truncate_with_size_parameter(self): # A SpooledTemporaryFile can be truncated to zero size f = tempfile.SpooledTemporaryFile(max_size=10) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 92398300f26577..1c540bed33c71e 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -361,9 +361,6 @@ def test_asctime(self): def test_asctime_bounding_check(self): self._bounds_checking(time.asctime) - @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" - ) def test_ctime(self): t = time.mktime((1973, 9, 16, 1, 3, 52, 0, 0, -1)) self.assertEqual(time.ctime(t), 'Sun Sep 16 01:03:52 1973') @@ -746,9 +743,6 @@ class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): class TestPytime(unittest.TestCase): @skip_if_buggy_ucrt_strfptime @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") - @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" - ) def test_localtime_timezone(self): # Get the localtime and examine it for the offset and zone. diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 25c16e3a0b7e43..4a067d714e12e3 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -125,8 +125,8 @@ def test_open(self): # open(), os.stat(), etc. don't raise any exception. @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "test fails on Emscripten/WASI when host platform is macOS." + support.is_wasi, + "test fails on WASI when host platform is macOS." ) def test_normalize(self): files = set(self.files) From b9a492b809d8765ee365a5dd3c6ba4e5130a80af Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 17 Dec 2024 16:18:33 +0800 Subject: [PATCH 223/427] gh-127085: Add a test skip if multiprocessing isn't available (#128019) Add a test skip if multiprocessing isn't available. --- Lib/test/test_memoryview.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 856e17918048e4..61b068c630c7ce 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -739,7 +739,10 @@ def test_picklebuffer_reference_loop(self): class RacingTest(unittest.TestCase): def test_racing_getbuf_and_releasebuf(self): """Repeatly access the memoryview for racing.""" - from multiprocessing.managers import SharedMemoryManager + try: + from multiprocessing.managers import SharedMemoryManager + except ImportError: + self.skipTest("Test requires multiprocessing") from threading import Thread n = 100 From 401bfc69d1258fde07141cd1a12af00aa44a8e0f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:49:37 +0200 Subject: [PATCH 224/427] Python 3.14.0a3 --- Doc/c-api/long.rst | 4 +- Doc/library/ctypes.rst | 4 +- Doc/library/errno.rst | 2 +- Doc/library/select.rst | 2 +- Doc/library/stdtypes.rst | 4 +- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 57 +- Misc/NEWS.d/3.14.0a3.rst | 1152 +++++++++++++++++ ...-11-20-17-12-40.gh-issue-126898.I2zILt.rst | 1 - ...-11-22-08-46-46.gh-issue-115869.UVLSKd.rst | 1 - ...-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst | 2 - ...-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst | 1 - ...-12-12-17-21-45.gh-issue-127865.30GDzs.rst | 1 - ...-07-03-13-39-13.gh-issue-121058.MKi1MV.rst | 2 - ...-07-03-17-26-53.gh-issue-102471.XpmKYk.rst | 10 - ...-08-12-10-15-19.gh-issue-109523.S2c3fi.rst | 1 - ...-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst | 6 - ...-11-26-22-06-10.gh-issue-127314.SsRrIu.rst | 2 - ...-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst | 6 - ...-12-06-16-53-34.gh-issue-127691.k_Jitp.rst | 3 - ...-12-10-14-25-22.gh-issue-127791.YRw4GU.rst | 2 - ...-12-16-07-12-15.gh-issue-127896.HmI9pk.rst | 2 - ...-09-22-21-01-56.gh-issue-109746.32MHt9.rst | 1 - ...-06-04-08-26-25.gh-issue-120010._z-AWz.rst | 2 - ...4-08-03-14-02-27.gh-issue-69639.mW3iKq.rst | 2 - ...-09-25-21-50-23.gh-issue-124470.pFr3_d.rst | 1 - ...-10-14-12-34-51.gh-issue-125420.jABXoZ.rst | 2 - ...-10-14-13-28-16.gh-issue-125420.hNKixM.rst | 2 - ...-10-27-04-47-28.gh-issue-126024.XCQSqT.rst | 2 - ...-11-07-21-48-23.gh-issue-126091.ETaRGE.rst | 2 - ...-11-15-16-39-37.gh-issue-126892.QR6Yo3.rst | 2 - ...-11-16-11-11-35.gh-issue-126881.ijofLZ.rst | 1 - ...-11-16-22-37-46.gh-issue-126868.yOoHSY.rst | 1 - ...-11-17-21-35-55.gh-issue-126937.qluVM0.rst | 3 - ...-11-18-23-18-17.gh-issue-126980.r8QHdi.rst | 3 - ...-11-19-17-17-32.gh-issue-127010.9Cl4bb.rst | 4 - ...-11-19-21-49-58.gh-issue-127020.5vvI17.rst | 4 - ...-11-21-16-13-52.gh-issue-126491.0YvL94.rst | 4 - ...-11-23-04-54-42.gh-issue-127133.WMoJjF.rst | 6 - ...-11-24-07-01-28.gh-issue-113841.WFg-Bu.rst | 2 - ...-11-25-05-15-21.gh-issue-127238.O8wkH-.rst | 1 - ...-11-30-23-35-45.gh-issue-127085.KLKylb.rst | 1 - ...-12-03-21-07-06.gh-issue-127536.3jMMrT.rst | 2 - ...-12-04-09-52-08.gh-issue-127434.RjkGT_.rst | 1 - ...-12-05-19-25-00.gh-issue-127582.ogUY2a.rst | 2 - ...-12-06-01-09-40.gh-issue-127651.80cm6j.rst | 1 - ...-12-07-13-06-09.gh-issue-127599.tXCZb_.rst | 2 - ...-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst | 3 - ...-12-10-21-08-05.gh-issue-127740.0tWC9h.rst | 3 - ...-11-27-22-56-48.gh-issue-127347.xyddWS.rst | 1 - ...2-11-10-17-16-45.gh-issue-97514.kzA0zl.rst | 10 - ...3-02-15-23-54-42.gh-issue-88110.KU6erv.rst | 2 - ...-07-25-18-06-51.gh-issue-122288.-_xxOR.rst | 2 - ...-07-29-15-20-30.gh-issue-122356.wKCmFx.rst | 3 - ...-07-30-11-37-40.gh-issue-122431.lAzVtu.rst | 1 - ...-08-27-18-58-01.gh-issue-123401.t4-FpI.rst | 3 - ...-09-13-18-24-27.gh-issue-124008.XaiPQx.rst | 2 - ...4-10-23-20-05-54.gh-issue-86463.jvFTI_.rst | 2 - ...-10-28-19-49-18.gh-issue-118201.v41XXh.rst | 2 - ...4-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst | 1 - ...-11-12-13-14-47.gh-issue-126727.5Eqfqd.rst | 3 - ...-11-12-20-05-09.gh-issue-126601.Nj7bA9.rst | 3 - ...-11-13-10-44-25.gh-issue-126775.a3ubjh.rst | 1 - ...-11-13-19-15-18.gh-issue-126780.ZZqJvI.rst | 1 - ...4-11-15-01-50-36.gh-issue-85168.bP8VIN.rst | 4 - ...-11-16-10-52-48.gh-issue-126899.GFnfBt.rst | 2 - ...-11-18-16-43-11.gh-issue-126946.52Ou-B.rst | 3 - ...-11-18-19-03-46.gh-issue-126947.NiDYUe.rst | 2 - ...-11-18-22-02-47.gh-issue-118761.GQKD_J.rst | 2 - ...-11-18-23-18-27.gh-issue-112192.DRdRgP.rst | 1 - ...-11-18-23-42-06.gh-issue-126985.7XplY9.rst | 3 - ...-11-19-14-34-05.gh-issue-126615.LOskwi.rst | 2 - ...-11-20-08-54-11.gh-issue-126618.ef_53g.rst | 2 - ...-11-20-11-37-08.gh-issue-126316.ElkZmE.rst | 2 - ...-11-20-16-58-59.gh-issue-126997.0PI41Y.rst | 3 - ...-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst | 1 - ...-11-21-06-03-46.gh-issue-127090.yUYwdh.rst | 3 - ...-11-21-16-23-16.gh-issue-127065.cfL1zd.rst | 2 - ...-11-22-02-31-55.gh-issue-126766.jfkhBH.rst | 2 - ...-11-22-03-40-02.gh-issue-127078.gI_PaP.rst | 2 - ...-11-22-04-49-31.gh-issue-125866.TUtvPK.rst | 2 - ...-11-22-09-23-41.gh-issue-122273.H8M6fd.rst | 1 - ...-11-22-10-42-34.gh-issue-127035.UnbDlr.rst | 4 - ...-11-23-00-17-29.gh-issue-127221.OSXdFE.rst | 1 - ...-11-23-12-25-06.gh-issue-125866.wEOP66.rst | 5 - ...-11-24-12-41-31.gh-issue-127217.UAXGFr.rst | 2 - ...-11-24-14-20-17.gh-issue-127182.WmfY2g.rst | 2 - ...-11-25-15-02-44.gh-issue-127255.UXeljc.rst | 2 - ...-11-25-19-04-10.gh-issue-127072.-c284K.rst | 1 - ...-11-26-17-42-00.gh-issue-127178.U8hxjc.rst | 4 - ...-11-27-14-06-35.gh-issue-123967.wxUmnW.rst | 2 - ...-11-27-14-23-02.gh-issue-127331.9sNEC9.rst | 1 - ...-11-27-16-06-10.gh-issue-127303.asqkgh.rst | 1 - ...4-11-27-17-04-38.gh-issue-59705.sAGyvs.rst | 2 - ...-11-28-14-14-46.gh-issue-127257.n6-jU9.rst | 2 - ...-11-29-00-15-59.gh-issue-125413.WCN0vv.rst | 3 - ...-11-29-14-45-26.gh-issue-127413.z11AUc.rst | 2 - ...-11-29-23-02-43.gh-issue-127429.dQf2w4.rst | 3 - ...-11-30-21-46-15.gh-issue-127321.M78fBv.rst | 1 - ...-12-01-22-28-41.gh-issue-127065.tFpRer.rst | 1 - ...-12-01-23-18-43.gh-issue-127481.K36AoP.rst | 1 - ...4-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst | 2 - ...-12-04-15-04-12.gh-issue-126821.lKCLVV.rst | 2 - ...-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst | 2 - ...-12-06-17-28-55.gh-issue-127610.ctv_NP.rst | 3 - ...-12-07-15-28-31.gh-issue-127718.9dpLfi.rst | 1 - ...-12-07-23-06-44.gh-issue-126789.4dqfV1.rst | 5 - ...-12-08-08-36-18.gh-issue-127732.UEKxoa.rst | 1 - ...-12-12-16-59-42.gh-issue-127870._NFG-3.rst | 2 - ...-12-13-22-20-54.gh-issue-126907.fWRL_R.rst | 2 - ...-12-05-21-35-19.gh-issue-127655.xpPoOf.rst | 1 - ...-11-20-18-49-01.gh-issue-127076.DHnXxo.rst | 2 - ...-11-21-02-03-48.gh-issue-127076.a3avV1.rst | 1 - ...-12-04-15-03-24.gh-issue-126925.uxAMK-.rst | 2 - ...-12-09-12-35-44.gh-issue-127637.KLx-9I.rst | 1 - ...-12-13-13-41-34.gh-issue-127906.NuRHlB.rst | 1 - ...-11-16-20-47-20.gh-issue-126700.ayrHv4.rst | 1 - ...-10-31-09-46-53.gh-issue-125729.KdKVLa.rst | 1 - ...-11-28-15-55-48.gh-issue-127353.i-XOXg.rst | 2 - README.rst | 2 +- 120 files changed, 1200 insertions(+), 279 deletions(-) create mode 100644 Misc/NEWS.d/3.14.0a3.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-11-20-17-12-40.gh-issue-126898.I2zILt.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-07-03-13-39-13.gh-issue-121058.MKi1MV.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-12-06-16-53-34.gh-issue-127691.k_Jitp.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst delete mode 100644 Misc/NEWS.d/next/C_API/2024-12-16-07-12-15.gh-issue-127896.HmI9pk.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2023-09-22-21-01-56.gh-issue-109746.32MHt9.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-08-03-14-02-27.gh-issue-69639.mW3iKq.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-21-50-23.gh-issue-124470.pFr3_d.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-07-21-48-23.gh-issue-126091.ETaRGE.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-15-16-39-37.gh-issue-126892.QR6Yo3.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-23-18-17.gh-issue-126980.r8QHdi.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-19-17-17-32.gh-issue-127010.9Cl4bb.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-19-21-49-58.gh-issue-127020.5vvI17.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-23-04-54-42.gh-issue-127133.WMoJjF.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-24-07-01-28.gh-issue-113841.WFg-Bu.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-25-05-15-21.gh-issue-127238.O8wkH-.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-07-13-06-09.gh-issue-127599.tXCZb_.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-11-10-17-16-45.gh-issue-97514.kzA0zl.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-07-25-18-06-51.gh-issue-122288.-_xxOR.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-07-29-15-20-30.gh-issue-122356.wKCmFx.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-09-13-18-24-27.gh-issue-124008.XaiPQx.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-10-23-20-05-54.gh-issue-86463.jvFTI_.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-10-28-19-49-18.gh-issue-118201.v41XXh.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-12-13-14-47.gh-issue-126727.5Eqfqd.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-12-20-05-09.gh-issue-126601.Nj7bA9.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-13-19-15-18.gh-issue-126780.ZZqJvI.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-15-01-50-36.gh-issue-85168.bP8VIN.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-18-16-43-11.gh-issue-126946.52Ou-B.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-18-19-03-46.gh-issue-126947.NiDYUe.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-18-22-02-47.gh-issue-118761.GQKD_J.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-19-14-34-05.gh-issue-126615.LOskwi.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-20-11-37-08.gh-issue-126316.ElkZmE.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-20-16-58-59.gh-issue-126997.0PI41Y.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-21-16-23-16.gh-issue-127065.cfL1zd.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-22-02-31-55.gh-issue-126766.jfkhBH.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-22-03-40-02.gh-issue-127078.gI_PaP.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-22-04-49-31.gh-issue-125866.TUtvPK.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-22-09-23-41.gh-issue-122273.H8M6fd.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-22-10-42-34.gh-issue-127035.UnbDlr.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-23-12-25-06.gh-issue-125866.wEOP66.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-24-12-41-31.gh-issue-127217.UAXGFr.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-24-14-20-17.gh-issue-127182.WmfY2g.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-25-19-04-10.gh-issue-127072.-c284K.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-27-14-06-35.gh-issue-123967.wxUmnW.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-27-14-23-02.gh-issue-127331.9sNEC9.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-12-16-59-42.gh-issue-127870._NFG-3.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-13-22-20-54.gh-issue-126907.fWRL_R.rst delete mode 100644 Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-11-20-18-49-01.gh-issue-127076.DHnXxo.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-11-21-02-03-48.gh-issue-127076.a3avV1.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-12-13-13-41-34.gh-issue-127906.NuRHlB.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2024-11-16-20-47-20.gh-issue-126700.ayrHv4.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-10-31-09-46-53.gh-issue-125729.KdKVLa.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f48cd07a979f56..084ba513493ffe 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -657,7 +657,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Export API ^^^^^^^^^^ -.. versionadded:: next +.. versionadded:: 3.14 .. c:struct:: PyLongLayout @@ -769,7 +769,7 @@ PyLongWriter API The :c:type:`PyLongWriter` API can be used to import an integer. -.. versionadded:: next +.. versionadded:: 3.14 .. c:struct:: PyLongWriter diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index bd88fa377fb39d..09692e56d29a39 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1964,7 +1964,7 @@ Utility functions .. availability:: Windows - .. versionadded:: next + .. versionadded:: 3.14 .. function:: cast(obj, type) @@ -2825,4 +2825,4 @@ Exceptions .. availability:: Windows - .. versionadded:: next + .. versionadded:: 3.14 diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 824d489818fac9..d8033663ea8eac 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -617,7 +617,7 @@ defined by the module. The specific list of defined symbols is available as Memory page has hardware error. - .. versionadded:: next + .. versionadded:: 3.14 .. data:: EALREADY diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 4fcff9198944a8..457970aed2dc73 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -324,7 +324,7 @@ Edge and Level Trigger Polling (epoll) Objects :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 or later. - .. versionadded:: next + .. versionadded:: 3.14 :const:`EPOLLWAKEUP` was added. It's only supported by Linux Kernel 3.5 or later. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index d5f7714ced6873..191827526e890f 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4153,7 +4153,7 @@ copying. Count the number of occurrences of *value*. - .. versionadded:: next + .. versionadded:: 3.14 .. method:: index(value, start=0, stop=sys.maxsize, /) @@ -4162,7 +4162,7 @@ copying. Raises a :exc:`ValueError` if *value* cannot be found. - .. versionadded:: next + .. versionadded:: 3.14 There are also several readonly attributes available: diff --git a/Include/patchlevel.h b/Include/patchlevel.h index e99c3a66f84e4f..1e9df2d9ebd07d 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 14 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 2 +#define PY_RELEASE_SERIAL 3 /* Version as a string */ -#define PY_VERSION "3.14.0a2+" +#define PY_VERSION "3.14.0a3" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index f73e55d77311ae..aebcef2b81d43d 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Nov 19 16:52:22 2024 +# Autogenerated by Sphinx on Tue Dec 17 11:49:52 2024 # as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' @@ -1312,15 +1312,19 @@ 'The arguments must either both be numbers, or one argument must be ' 'an\n' 'integer and the other must be a sequence. In the former case, the\n' - 'numbers are converted to a common type and then multiplied ' - 'together.\n' - 'In the latter case, sequence repetition is performed; a negative\n' - 'repetition factor yields an empty sequence.\n' + 'numbers are converted to a common real type and then multiplied\n' + 'together. In the latter case, sequence repetition is performed; ' + 'a\n' + 'negative repetition factor yields an empty sequence.\n' '\n' 'This operation can be customized using the special "__mul__()" ' 'and\n' '"__rmul__()" methods.\n' '\n' + 'Changed in version 3.14: If only one operand is a complex number, ' + 'the\n' + 'other operand is converted to a floating-point number.\n' + '\n' 'The "@" (at) operator is intended to be used for matrix\n' 'multiplication. No builtin Python types implement this operator.\n' '\n' @@ -1391,21 +1395,30 @@ 'arguments must either both be numbers or both be sequences of the ' 'same\n' 'type. In the former case, the numbers are converted to a common ' - 'type\n' - 'and then added together. In the latter case, the sequences are\n' + 'real\n' + 'type and then added together. In the latter case, the sequences ' + 'are\n' 'concatenated.\n' '\n' 'This operation can be customized using the special "__add__()" ' 'and\n' '"__radd__()" methods.\n' '\n' + 'Changed in version 3.14: If only one operand is a complex number, ' + 'the\n' + 'other operand is converted to a floating-point number.\n' + '\n' 'The "-" (subtraction) operator yields the difference of its ' 'arguments.\n' - 'The numeric arguments are first converted to a common type.\n' + 'The numeric arguments are first converted to a common real type.\n' '\n' 'This operation can be customized using the special "__sub__()" ' 'and\n' - '"__rsub__()" methods.\n', + '"__rsub__()" methods.\n' + '\n' + 'Changed in version 3.14: If only one operand is a complex number, ' + 'the\n' + 'other operand is converted to a floating-point number.\n', 'bitwise': 'Binary bitwise operations\n' '*************************\n' '\n' @@ -4561,18 +4574,18 @@ '\n' 'When a description of an arithmetic operator below uses the ' 'phrase\n' - '“the numeric arguments are converted to a common type”, this ' - 'means\n' - 'that the operator implementation for built-in types works as ' + '“the numeric arguments are converted to a common real type”, ' + 'this\n' + 'means that the operator implementation for built-in types ' + 'works as\n' 'follows:\n' '\n' - '* If either argument is a complex number, the other is ' - 'converted to\n' - ' complex;\n' + '* If both arguments are complex numbers, no conversion is ' + 'performed;\n' '\n' - '* otherwise, if either argument is a floating-point number, ' - 'the other\n' - ' is converted to floating point;\n' + '* if either argument is a complex or a floating-point number, ' + 'the\n' + ' other is converted to a floating-point number;\n' '\n' '* otherwise, both must be integers and no conversion is ' 'necessary.\n' @@ -7144,8 +7157,12 @@ 'trailing zeros are not removed from the result.\n' '\n' 'The "\',\'" option signals the use of a comma for a ' - 'thousands separator.\n' - 'For a locale aware separator, use the "\'n\'" integer ' + 'thousands separator\n' + 'for floating-point presentation types and for integer ' + 'presentation\n' + 'type "\'d\'". For other presentation types, this option is ' + 'an error. For\n' + 'a locale aware separator, use the "\'n\'" integer ' 'presentation type\n' 'instead.\n' '\n' diff --git a/Misc/NEWS.d/3.14.0a3.rst b/Misc/NEWS.d/3.14.0a3.rst new file mode 100644 index 00000000000000..8393be8909ff8f --- /dev/null +++ b/Misc/NEWS.d/3.14.0a3.rst @@ -0,0 +1,1152 @@ +.. date: 2024-11-28-15-55-48 +.. gh-issue: 127353 +.. nonce: i-XOXg +.. release date: 2024-12-17 +.. section: Windows + +Allow to force color output on Windows using environment variables. Patch by +Andrey Efremov. + +.. + +.. date: 2024-10-31-09-46-53 +.. gh-issue: 125729 +.. nonce: KdKVLa +.. section: Windows + +Makes the presence of the :mod:`turtle` module dependent on the Tcl/Tk +installer option. Previously, the module was always installed but would be +unusable without Tcl/Tk. + +.. + +.. date: 2024-11-16-20-47-20 +.. gh-issue: 126700 +.. nonce: ayrHv4 +.. section: Tools/Demos + +Add support for multi-argument :mod:`gettext` functions in +:program:`pygettext.py`. + +.. + +.. date: 2024-12-13-13-41-34 +.. gh-issue: 127906 +.. nonce: NuRHlB +.. section: Tests + +Test the limited C API in test_cppext. Patch by Victor Stinner. + +.. + +.. date: 2024-12-09-12-35-44 +.. gh-issue: 127637 +.. nonce: KLx-9I +.. section: Tests + +Add tests for the :mod:`dis` command-line interface. Patch by Bénédikt Tran. + +.. + +.. date: 2024-12-04-15-03-24 +.. gh-issue: 126925 +.. nonce: uxAMK- +.. section: Tests + +iOS test results are now streamed during test execution, and the deprecated +xcresulttool is no longer used. + +.. + +.. date: 2024-11-21-02-03-48 +.. gh-issue: 127076 +.. nonce: a3avV1 +.. section: Tests + +Disable strace based system call tests when LD_PRELOAD is set. + +.. + +.. date: 2024-11-20-18-49-01 +.. gh-issue: 127076 +.. nonce: DHnXxo +.. section: Tests + +Filter out memory-related ``mmap``, ``munmap``, and ``mprotect`` calls from +file-related ones when testing :mod:`io` behavior using strace. + +.. + +.. date: 2024-12-05-21-35-19 +.. gh-issue: 127655 +.. nonce: xpPoOf +.. section: Security + +Fixed the :class:`!asyncio.selector_events._SelectorSocketTransport` +transport not pausing writes for the protocol when the buffer reaches the +high water mark when using :meth:`asyncio.WriteTransport.writelines`. + +.. + +.. date: 2024-12-13-22-20-54 +.. gh-issue: 126907 +.. nonce: fWRL_R +.. section: Library + +Fix crash when using :mod:`atexit` concurrently on the :term:`free-threaded +` build. + +.. + +.. date: 2024-12-12-16-59-42 +.. gh-issue: 127870 +.. nonce: _NFG-3 +.. section: Library + +Detect recursive calls in ctypes ``_as_parameter_`` handling. Patch by +Victor Stinner. + +.. + +.. date: 2024-12-08-08-36-18 +.. gh-issue: 127732 +.. nonce: UEKxoa +.. section: Library + +The :mod:`platform` module now correctly detects Windows Server 2025. + +.. + +.. date: 2024-12-07-23-06-44 +.. gh-issue: 126789 +.. nonce: 4dqfV1 +.. section: Library + +Fixed :func:`sysconfig.get_config_vars`, :func:`sysconfig.get_paths`, and +siblings, returning outdated cached data if the value of :data:`sys.prefix` +or :data:`sys.exec_prefix` changes. Overwriting :data:`sys.prefix` or +:data:`sys.exec_prefix` still is discouraged, as that might break other +parts of the code. + +.. + +.. date: 2024-12-07-15-28-31 +.. gh-issue: 127718 +.. nonce: 9dpLfi +.. section: Library + +Add colour to :mod:`test.regrtest` output. Patch by Hugo van Kemenade. + +.. + +.. date: 2024-12-06-17-28-55 +.. gh-issue: 127610 +.. nonce: ctv_NP +.. section: Library + +Added validation for more than one var-positional or var-keyword parameters +in :class:`inspect.Signature`. Patch by Maxim Ageev. + +.. + +.. date: 2024-12-05-10-14-52 +.. gh-issue: 127627 +.. nonce: fgCHOZ +.. section: Library + +Added ``posix._emscripten_debugger()`` to help with debugging the test suite +on the Emscripten target. + +.. + +.. date: 2024-12-04-15-04-12 +.. gh-issue: 126821 +.. nonce: lKCLVV +.. section: Library + +macOS and iOS apps can now choose to redirect stdout and stderr to the +system log during interpreter configuration. + +.. + +.. date: 2024-12-04-11-01-16 +.. gh-issue: 93312 +.. nonce: 9sB-Qw +.. section: Library + +Include ```` to get ``os.PIDFD_NONBLOCK`` constant. Patch by +Victor Stinner. + +.. + +.. date: 2024-12-01-23-18-43 +.. gh-issue: 127481 +.. nonce: K36AoP +.. section: Library + +Add the ``EPOLLWAKEUP`` constant to the :mod:`select` module. + +.. + +.. date: 2024-12-01-22-28-41 +.. gh-issue: 127065 +.. nonce: tFpRer +.. section: Library + +Make :func:`operator.methodcaller` thread-safe and re-entrant safe. + +.. + +.. date: 2024-11-30-21-46-15 +.. gh-issue: 127321 +.. nonce: M78fBv +.. section: Library + +:func:`pdb.set_trace` will not stop at an opcode that does not have an +associated line number anymore. + +.. + +.. date: 2024-11-29-23-02-43 +.. gh-issue: 127429 +.. nonce: dQf2w4 +.. section: Library + +Fixed bug where, on cross-builds, the :mod:`sysconfig` POSIX data was being +generated with the host Python's ``Makefile``. The data is now generated +from current build's ``Makefile``. + +.. + +.. date: 2024-11-29-14-45-26 +.. gh-issue: 127413 +.. nonce: z11AUc +.. section: Library + +Add the :option:`dis --specialized` command-line option to show specialized +bytecode. Patch by Bénédikt Tran. + +.. + +.. date: 2024-11-29-00-15-59 +.. gh-issue: 125413 +.. nonce: WCN0vv +.. section: Library + +Revert addition of :meth:`!pathlib.Path.scandir`. This method was added in +3.14.0a2. The optimizations remain for file system paths, but other +subclasses should only have to implement :meth:`pathlib.Path.iterdir`. + +.. + +.. date: 2024-11-28-14-14-46 +.. gh-issue: 127257 +.. nonce: n6-jU9 +.. section: Library + +In :mod:`ssl`, system call failures that OpenSSL reports using +``ERR_LIB_SYS`` are now raised as :exc:`OSError`. + +.. + +.. date: 2024-11-27-17-04-38 +.. gh-issue: 59705 +.. nonce: sAGyvs +.. section: Library + +On Linux, :class:`threading.Thread` now sets the thread name to the +operating system. Patch by Victor Stinner. + +.. + +.. date: 2024-11-27-16-06-10 +.. gh-issue: 127303 +.. nonce: asqkgh +.. section: Library + +Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`. + +.. + +.. date: 2024-11-27-14-23-02 +.. gh-issue: 127331 +.. nonce: 9sNEC9 +.. section: Library + +:mod:`ssl` can show descriptions for errors added in OpenSSL 3.4. + +.. + +.. date: 2024-11-27-14-06-35 +.. gh-issue: 123967 +.. nonce: wxUmnW +.. section: Library + +Fix faulthandler for trampoline frames. If the top-most frame is a +trampoline frame, skip it. Patch by Victor Stinner. + +.. + +.. date: 2024-11-26-17-42-00 +.. gh-issue: 127178 +.. nonce: U8hxjc +.. section: Library + +A ``_sysconfig_vars_(...).json`` file is now shipped in the standard library +directory. It contains the output of :func:`sysconfig.get_config_vars` on +the default environment encoded as JSON data. This is an implementation +detail, and may change at any time. + +.. + +.. date: 2024-11-25-19-04-10 +.. gh-issue: 127072 +.. nonce: -c284K +.. section: Library + +Remove outdated ``socket.NETLINK_*`` constants not present in Linux kernels +beyond 2.6.17. + +.. + +.. date: 2024-11-25-15-02-44 +.. gh-issue: 127255 +.. nonce: UXeljc +.. section: Library + +The :func:`~ctypes.CopyComPointer` function is now public. Previously, this +was private and only available in ``_ctypes``. + +.. + +.. date: 2024-11-24-14-20-17 +.. gh-issue: 127182 +.. nonce: WmfY2g +.. section: Library + +Fix :meth:`!io.StringIO.__setstate__` crash, when :const:`None` was passed +as the first value. + +.. + +.. date: 2024-11-24-12-41-31 +.. gh-issue: 127217 +.. nonce: UAXGFr +.. section: Library + +Fix :func:`urllib.request.pathname2url` for paths starting with multiple +slashes on Posix. + +.. + +.. date: 2024-11-23-12-25-06 +.. gh-issue: 125866 +.. nonce: wEOP66 +.. section: Library + +:func:`urllib.request.pathname2url` now adds an empty authority when +generating a URL for a path that begins with exactly one slash. For example, +the path ``/etc/hosts`` is converted to the scheme-less URL +``///etc/hosts``. As a result of this change, URLs without authorities are +only generated for relative paths. + +.. + +.. date: 2024-11-23-00-17-29 +.. gh-issue: 127221 +.. nonce: OSXdFE +.. section: Library + +Add colour to :mod:`unittest` output. Patch by Hugo van Kemenade. + +.. + +.. date: 2024-11-22-10-42-34 +.. gh-issue: 127035 +.. nonce: UnbDlr +.. section: Library + +Fix :mod:`shutil.which` on Windows. Now it looks at direct match if and only +if the command ends with a PATHEXT extension or X_OK is not in mode. Support +extensionless files if "." is in PATHEXT. Support PATHEXT extensions that +end with a dot. + +.. + +.. date: 2024-11-22-09-23-41 +.. gh-issue: 122273 +.. nonce: H8M6fd +.. section: Library + +Support PyREPL history on Windows. Patch by devdanzin and Victor Stinner. + +.. + +.. date: 2024-11-22-04-49-31 +.. gh-issue: 125866 +.. nonce: TUtvPK +.. section: Library + +:func:`urllib.request.pathname2url` and :func:`~urllib.request.url2pathname` +no longer convert Windows drive letters to uppercase. + +.. + +.. date: 2024-11-22-03-40-02 +.. gh-issue: 127078 +.. nonce: gI_PaP +.. section: Library + +Fix issue where :func:`urllib.request.url2pathname` failed to discard an +extra slash before a UNC drive in the URL path on Windows. + +.. + +.. date: 2024-11-22-02-31-55 +.. gh-issue: 126766 +.. nonce: jfkhBH +.. section: Library + +Fix issue where :func:`urllib.request.url2pathname` failed to discard any +'localhost' authority present in the URL. + +.. + +.. date: 2024-11-21-16-23-16 +.. gh-issue: 127065 +.. nonce: cfL1zd +.. section: Library + +Fix crash when calling a :func:`operator.methodcaller` instance from +multiple threads in the free threading build. + +.. + +.. date: 2024-11-21-06-03-46 +.. gh-issue: 127090 +.. nonce: yUYwdh +.. section: Library + +Fix value of :attr:`urllib.response.addinfourl.url` for ``file:`` URLs that +express relative paths and absolute Windows paths. The canonical URL +generated by :func:`urllib.request.pathname2url` is now used. + +.. + +.. date: 2024-11-20-21-20-56 +.. gh-issue: 126992 +.. nonce: RbU0FZ +.. section: Library + +Fix LONG and INT opcodes to only use base 10 for string to integer +conversion in :mod:`pickle`. + +.. + +.. date: 2024-11-20-16-58-59 +.. gh-issue: 126997 +.. nonce: 0PI41Y +.. section: Library + +Fix support of STRING and GLOBAL opcodes with non-ASCII arguments in +:mod:`pickletools`. :func:`pickletools.dis` now outputs non-ASCII bytes in +STRING, BINSTRING and SHORT_BINSTRING arguments as escaped (``\xXX``). + +.. + +.. date: 2024-11-20-11-37-08 +.. gh-issue: 126316 +.. nonce: ElkZmE +.. section: Library + +:mod:`grp`: Make :func:`grp.getgrall` thread-safe by adding a mutex. Patch +by Victor Stinner. + +.. + +.. date: 2024-11-20-08-54-11 +.. gh-issue: 126618 +.. nonce: ef_53g +.. section: Library + +Fix the representation of :class:`itertools.count` objects when the count +value is :data:`sys.maxsize`. + +.. + +.. date: 2024-11-19-14-34-05 +.. gh-issue: 126615 +.. nonce: LOskwi +.. section: Library + +The :exc:`~ctypes.COMError` exception is now public. Previously, this was +private and only available in ``_ctypes``. + +.. + +.. date: 2024-11-18-23-42-06 +.. gh-issue: 126985 +.. nonce: 7XplY9 +.. section: Library + +When running under a virtual environment with the :mod:`site` disabled (see +:option:`-S`), :data:`sys.prefix` and :data:`sys.base_prefix` will now point +to the virtual environment, instead of the base installation. + +.. + +.. date: 2024-11-18-23-18-27 +.. gh-issue: 112192 +.. nonce: DRdRgP +.. section: Library + +In the :mod:`trace` module, increase the coverage precision (``cov%``) to +one decimal. + +.. + +.. date: 2024-11-18-22-02-47 +.. gh-issue: 118761 +.. nonce: GQKD_J +.. section: Library + +Improve import time of :mod:`mimetypes` by around 11-16 times. Patch by Hugo +van Kemenade. + +.. + +.. date: 2024-11-18-19-03-46 +.. gh-issue: 126947 +.. nonce: NiDYUe +.. section: Library + +Raise :exc:`TypeError` in :meth:`!_pydatetime.timedelta.__new__` if the +passed arguments are not :class:`int` or :class:`float`, so that the Python +implementation is in line with the C implementation. + +.. + +.. date: 2024-11-18-16-43-11 +.. gh-issue: 126946 +.. nonce: 52Ou-B +.. section: Library + +Improve the :exc:`~getopt.GetoptError` error message when a long option +prefix matches multiple accepted options in :func:`getopt.getopt` and +:func:`getopt.gnu_getopt`. + +.. + +.. date: 2024-11-16-10-52-48 +.. gh-issue: 126899 +.. nonce: GFnfBt +.. section: Library + +Make tkinter widget methods :meth:`!after` and :meth:`!after_idle` accept +arguments passed by keyword. + +.. + +.. date: 2024-11-15-01-50-36 +.. gh-issue: 85168 +.. nonce: bP8VIN +.. section: Library + +Fix issue where :func:`urllib.request.url2pathname` and +:func:`~urllib.request.pathname2url` always used UTF-8 when quoting and +unquoting file URIs. They now use the :term:`filesystem encoding and error +handler`. + +.. + +.. date: 2024-11-13-19-15-18 +.. gh-issue: 126780 +.. nonce: ZZqJvI +.. section: Library + +Fix :func:`os.path.normpath` for drive-relative paths on Windows. + +.. + +.. date: 2024-11-13-10-44-25 +.. gh-issue: 126775 +.. nonce: a3ubjh +.. section: Library + +Make :func:`linecache.checkcache` thread safe and GC re-entrancy safe. + +.. + +.. date: 2024-11-12-20-05-09 +.. gh-issue: 126601 +.. nonce: Nj7bA9 +.. section: Library + +Fix issue where :func:`urllib.request.pathname2url` raised :exc:`OSError` +when given a Windows path containing a colon character not following a drive +letter, such as before an NTFS alternate data stream. + +.. + +.. date: 2024-11-12-13-14-47 +.. gh-issue: 126727 +.. nonce: 5Eqfqd +.. section: Library + +``locale.nl_langinfo(locale.ERA)`` now returns multiple era description +segments separated by semicolons. Previously it only returned the first +segment on platforms with Glibc. + +.. + +.. date: 2024-11-04-22-02-30 +.. gh-issue: 85046 +.. nonce: Y5d_ZN +.. section: Library + +Add :data:`~errno.EHWPOISON` error code to :mod:`errno`. + +.. + +.. date: 2024-10-28-19-49-18 +.. gh-issue: 118201 +.. nonce: v41XXh +.. section: Library + +Fixed intermittent failures of :any:`os.confstr`, :any:`os.pathconf` and +:any:`os.sysconf` on iOS and Android. + +.. + +.. date: 2024-10-23-20-05-54 +.. gh-issue: 86463 +.. nonce: jvFTI_ +.. section: Library + +The ``usage`` parameter of :class:`argparse.ArgumentParser` no longer +affects the default value of the ``prog`` parameter in subparsers. + +.. + +.. date: 2024-09-13-18-24-27 +.. gh-issue: 124008 +.. nonce: XaiPQx +.. section: Library + +Fix possible crash (in debug build), incorrect output or returning incorrect +value from raw binary ``write()`` when writing to console on Windows. + +.. + +.. date: 2024-08-27-18-58-01 +.. gh-issue: 123401 +.. nonce: t4-FpI +.. section: Library + +The :mod:`http.cookies` module now supports parsing obsolete :rfc:`850` date +formats, in accordance with :rfc:`9110` requirements. Patch by Nano Zheng. + +.. + +.. date: 2024-07-30-11-37-40 +.. gh-issue: 122431 +.. nonce: lAzVtu +.. section: Library + +:func:`readline.append_history_file` now raises a :exc:`ValueError` when +given a negative value. + +.. + +.. date: 2024-07-29-15-20-30 +.. gh-issue: 122356 +.. nonce: wKCmFx +.. section: Library + +Guarantee that the position of a file-like object passed to +:func:`zipfile.is_zipfile` is left untouched after the call. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-07-25-18-06-51 +.. gh-issue: 122288 +.. nonce: -_xxOR +.. section: Library + +Improve the performances of :func:`fnmatch.translate` by a factor 1.7. Patch +by Bénédikt Tran. + +.. + +.. date: 2023-02-15-23-54-42 +.. gh-issue: 88110 +.. nonce: KU6erv +.. section: Library + +Fixed :class:`multiprocessing.Process` reporting a ``.exitcode`` of 1 even +on success when using the ``"fork"`` start method while using a +:class:`concurrent.futures.ThreadPoolExecutor`. + +.. + +.. date: 2022-11-10-17-16-45 +.. gh-issue: 97514 +.. nonce: kzA0zl +.. section: Library + +Authentication was added to the :mod:`multiprocessing` forkserver start +method control socket so that only processes with the authentication key +generated by the process that spawned the forkserver can control it. This +is an enhancement over the other :gh:`97514` fixes so that access is no +longer limited only by filesystem permissions. + +The file descriptor exchange of control pipes with the forked worker process +now requires an explicit acknowledgement byte to be sent over the socket +after the exchange on all forkserver supporting platforms. That makes +testing the above much easier. + +.. + +.. date: 2024-11-27-22-56-48 +.. gh-issue: 127347 +.. nonce: xyddWS +.. section: Documentation + +Publicly expose :func:`traceback.print_list` in :attr:`!traceback.__all__`. + +.. + +.. date: 2024-12-10-21-08-05 +.. gh-issue: 127740 +.. nonce: 0tWC9h +.. section: Core and Builtins + +Fix error message in :func:`bytes.fromhex` when given an odd number of +digits to properly indicate that an even number of hexadecimal digits is +required. + +.. + +.. date: 2024-12-09-11-29-10 +.. gh-issue: 127058 +.. nonce: pqtBcZ +.. section: Core and Builtins + +``PySequence_Tuple`` now creates the resulting tuple atomically, preventing +partially created tuples being visible to the garbage collector or through +``gc.get_referrers()`` + +.. + +.. date: 2024-12-07-13-06-09 +.. gh-issue: 127599 +.. nonce: tXCZb_ +.. section: Core and Builtins + +Fix statistics for increments of object reference counts (in particular, +when a reference count was increased by more than 1 in a single operation). + +.. + +.. date: 2024-12-06-01-09-40 +.. gh-issue: 127651 +.. nonce: 80cm6j +.. section: Core and Builtins + +When raising :exc:`ImportError` for missing symbols in ``from`` imports, use +``__file__`` in the error message if ``__spec__.origin`` is not a location + +.. + +.. date: 2024-12-05-19-25-00 +.. gh-issue: 127582 +.. nonce: ogUY2a +.. section: Core and Builtins + +Fix non-thread-safe object resurrection when calling finalizers and watcher +callbacks in the free threading build. + +.. + +.. date: 2024-12-04-09-52-08 +.. gh-issue: 127434 +.. nonce: RjkGT_ +.. section: Core and Builtins + +The iOS compiler shims can now accept arguments with spaces. + +.. + +.. date: 2024-12-03-21-07-06 +.. gh-issue: 127536 +.. nonce: 3jMMrT +.. section: Core and Builtins + +Add missing locks around some list assignment operations in the free +threading build. + +.. + +.. date: 2024-11-30-23-35-45 +.. gh-issue: 127085 +.. nonce: KLKylb +.. section: Core and Builtins + +Fix race when exporting a buffer from a :class:`memoryview` object on the +:term:`free-threaded ` build. + +.. + +.. date: 2024-11-25-05-15-21 +.. gh-issue: 127238 +.. nonce: O8wkH- +.. section: Core and Builtins + +Correct error message for :func:`sys.set_int_max_str_digits`. + +.. + +.. date: 2024-11-24-07-01-28 +.. gh-issue: 113841 +.. nonce: WFg-Bu +.. section: Core and Builtins + +Fix possible undefined behavior division by zero in :class:`complex`'s +:c:func:`_Py_c_pow`. + +.. + +.. date: 2024-11-23-04-54-42 +.. gh-issue: 127133 +.. nonce: WMoJjF +.. section: Core and Builtins + +Calling :meth:`argparse.ArgumentParser.add_argument_group` on an argument +group, and calling :meth:`argparse.ArgumentParser.add_argument_group` or +:meth:`argparse.ArgumentParser.add_mutually_exclusive_group` on a mutually +exclusive group now raise exceptions. This nesting was never supported, +often failed to work correctly, and was unintentionally exposed through +inheritance. This functionality has been deprecated since Python 3.11. + +.. + +.. date: 2024-11-21-16-13-52 +.. gh-issue: 126491 +.. nonce: 0YvL94 +.. section: Core and Builtins + +Add a marking phase to the GC. All objects that can be transitively reached +from builtin modules or the stacks are marked as reachable before cycle +detection. This reduces the amount of work done by the GC by approximately +half. + +.. + +.. date: 2024-11-19-21-49-58 +.. gh-issue: 127020 +.. nonce: 5vvI17 +.. section: Core and Builtins + +Fix a crash in the free threading build when :c:func:`PyCode_GetCode`, +:c:func:`PyCode_GetVarnames`, :c:func:`PyCode_GetCellvars`, or +:c:func:`PyCode_GetFreevars` were called from multiple threads at the same +time. + +.. + +.. date: 2024-11-19-17-17-32 +.. gh-issue: 127010 +.. nonce: 9Cl4bb +.. section: Core and Builtins + +Simplify GC tracking of dictionaries. All dictionaries are tracked when +created, rather than being lazily tracked when a trackable object was added +to them. This simplifies the code considerably and results in a slight +speedup. + +.. + +.. date: 2024-11-18-23-18-17 +.. gh-issue: 126980 +.. nonce: r8QHdi +.. section: Core and Builtins + +Fix :meth:`~object.__buffer__` of :class:`bytearray` crashing when +:attr:`~inspect.BufferFlags.READ` or :attr:`~inspect.BufferFlags.WRITE` are +passed as flags. + +.. + +.. date: 2024-11-17-21-35-55 +.. gh-issue: 126937 +.. nonce: qluVM0 +.. section: Core and Builtins + +Fix :exc:`TypeError` when a :class:`ctypes.Structure` has a field size that +doesn't fit into an unsigned 16-bit integer. Instead, the maximum number of +*bits* is :data:`sys.maxsize`. + +.. + +.. date: 2024-11-16-22-37-46 +.. gh-issue: 126868 +.. nonce: yOoHSY +.. section: Core and Builtins + +Increase performance of :class:`int` by adding a freelist for compact ints. + +.. + +.. date: 2024-11-16-11-11-35 +.. gh-issue: 126881 +.. nonce: ijofLZ +.. section: Core and Builtins + +Fix crash in finalization of dtoa state. Patch by Kumar Aditya. + +.. + +.. date: 2024-11-15-16-39-37 +.. gh-issue: 126892 +.. nonce: QR6Yo3 +.. section: Core and Builtins + +Require cold or invalidated code to "warm up" before being JIT compiled +again. + +.. + +.. date: 2024-11-07-21-48-23 +.. gh-issue: 126091 +.. nonce: ETaRGE +.. section: Core and Builtins + +Ensure stack traces are complete when throwing into a generator chain that +ends in a custom generator. + +.. + +.. date: 2024-10-27-04-47-28 +.. gh-issue: 126024 +.. nonce: XCQSqT +.. section: Core and Builtins + +Optimize decoding of short UTF-8 sequences containing non-ASCII characters +by approximately 15%. + +.. + +.. date: 2024-10-14-13-28-16 +.. gh-issue: 125420 +.. nonce: hNKixM +.. section: Core and Builtins + +Add :meth:`memoryview.index` to :class:`memoryview` objects. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-10-14-12-34-51 +.. gh-issue: 125420 +.. nonce: jABXoZ +.. section: Core and Builtins + +Add :meth:`memoryview.count` to :class:`memoryview` objects. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-09-25-21-50-23 +.. gh-issue: 124470 +.. nonce: pFr3_d +.. section: Core and Builtins + +Fix crash in free-threaded builds when replacing object dictionary while +reading attribute on another thread + +.. + +.. date: 2024-08-03-14-02-27 +.. gh-issue: 69639 +.. nonce: mW3iKq +.. section: Core and Builtins + +Implement mixed-mode arithmetic rules combining real and complex numbers as +specified by C standards since C99. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-06-04-08-26-25 +.. gh-issue: 120010 +.. nonce: _z-AWz +.. section: Core and Builtins + +Correct invalid corner cases which resulted in ``(nan+nanj)`` output in +complex multiplication, e.g., ``(1e300+1j)*(nan+infj)``. Patch by Sergey B +Kirpichev. + +.. + +.. date: 2023-09-22-21-01-56 +.. gh-issue: 109746 +.. nonce: 32MHt9 +.. section: Core and Builtins + +If :func:`!_thread.start_new_thread` fails to start a new thread, it deletes +its state from interpreter and thus avoids its repeated cleanup on +finalization. + +.. + +.. date: 2024-12-16-07-12-15 +.. gh-issue: 127896 +.. nonce: HmI9pk +.. section: C API + +The previously undocumented function :c:func:`PySequence_In` is :term:`soft +deprecated`. Use :c:func:`PySequence_Contains` instead. + +.. + +.. date: 2024-12-10-14-25-22 +.. gh-issue: 127791 +.. nonce: YRw4GU +.. section: C API + +Fix loss of callbacks after more than one call to +:c:func:`PyUnstable_AtExit`. + +.. + +.. date: 2024-12-06-16-53-34 +.. gh-issue: 127691 +.. nonce: k_Jitp +.. section: C API + +The :ref:`Unicode Exception Objects ` C API now raises a +:exc:`TypeError` if its exception argument is not a :exc:`UnicodeError` +object. Patch by Bénédikt Tran. + +.. + +.. date: 2024-12-02-16-10-36 +.. gh-issue: 123378 +.. nonce: Q6YRwe +.. section: C API + +Ensure that the value of :attr:`UnicodeEncodeError.end ` +retrieved by :c:func:`PyUnicodeEncodeError_GetEnd` lies in ``[min(1, +objlen), max(min(1, objlen), objlen)]`` where *objlen* is the length of +:attr:`UnicodeEncodeError.object `. Similar arguments +apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` and +their corresponding C interface. Patch by Bénédikt Tran. + +.. + +.. date: 2024-11-26-22-06-10 +.. gh-issue: 127314 +.. nonce: SsRrIu +.. section: C API + +Improve error message when calling the C API without an active thread state +on the :term:`free-threaded ` build. + +.. + +.. date: 2024-08-27-09-07-56 +.. gh-issue: 123378 +.. nonce: JJ6n_u +.. section: C API + +Ensure that the value of :attr:`UnicodeEncodeError.start +` retrieved by :c:func:`PyUnicodeEncodeError_GetStart` +lies in ``[0, max(0, objlen - 1)]`` where *objlen* is the length of +:attr:`UnicodeEncodeError.object `. Similar arguments +apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` and +their corresponding C interface. Patch by Bénédikt Tran. + +.. + +.. date: 2024-08-12-10-15-19 +.. gh-issue: 109523 +.. nonce: S2c3fi +.. section: C API + +Reading text from a non-blocking stream with ``read`` may now raise a +:exc:`BlockingIOError` if the operation cannot immediately return bytes. + +.. + +.. date: 2024-07-03-17-26-53 +.. gh-issue: 102471 +.. nonce: XpmKYk +.. section: C API + +Add a new import and export API for Python :class:`int` objects +(:pep:`757`): + +* :c:func:`PyLong_GetNativeLayout`; +* :c:func:`PyLong_Export`; +* :c:func:`PyLong_FreeExport`; +* :c:func:`PyLongWriter_Create`; +* :c:func:`PyLongWriter_Finish`; +* :c:func:`PyLongWriter_Discard`. + +Patch by Victor Stinner. + +.. + +.. date: 2024-07-03-13-39-13 +.. gh-issue: 121058 +.. nonce: MKi1MV +.. section: C API + +``PyThreadState_Clear()`` now warns (and calls ``sys.excepthook``) if the +thread state still has an active exception. + +.. + +.. date: 2024-12-12-17-21-45 +.. gh-issue: 127865 +.. nonce: 30GDzs +.. section: Build + +Fix build failure on systems without thread-locals support. + +.. + +.. date: 2024-12-06-12-47-52 +.. gh-issue: 127629 +.. nonce: tD-ERQ +.. section: Build + +Emscripten builds now include ctypes support. + +.. + +.. date: 2024-11-30-16-36-09 +.. gh-issue: 127111 +.. nonce: QI9mMZ +.. section: Build + +Updated the Emscripten web example to use ES6 modules and be built into a +distinct ``web_example`` subfolder. + +.. + +.. date: 2024-11-22-08-46-46 +.. gh-issue: 115869 +.. nonce: UVLSKd +.. section: Build + +Make ``jit_stencils.h`` (which is produced during JIT builds) reproducible. + +.. + +.. date: 2024-11-20-17-12-40 +.. gh-issue: 126898 +.. nonce: I2zILt +.. section: Build + +The Emscripten build of Python is now based on ES6 modules. diff --git a/Misc/NEWS.d/next/Build/2024-11-20-17-12-40.gh-issue-126898.I2zILt.rst b/Misc/NEWS.d/next/Build/2024-11-20-17-12-40.gh-issue-126898.I2zILt.rst deleted file mode 100644 index 37783c4e890015..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-11-20-17-12-40.gh-issue-126898.I2zILt.rst +++ /dev/null @@ -1 +0,0 @@ -The Emscripten build of Python is now based on ES6 modules. diff --git a/Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst b/Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst deleted file mode 100644 index 9e8a078983f20b..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst +++ /dev/null @@ -1 +0,0 @@ -Make ``jit_stencils.h`` (which is produced during JIT builds) reproducible. diff --git a/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst deleted file mode 100644 index d90067cd3bfaa3..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Updated the Emscripten web example to use ES6 modules and be built into a -distinct ``web_example`` subfolder. diff --git a/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst b/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst deleted file mode 100644 index 52ee84f5d6f6c3..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst +++ /dev/null @@ -1 +0,0 @@ -Emscripten builds now include ctypes support. diff --git a/Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst b/Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst deleted file mode 100644 index 3fc1d8a1b51d30..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst +++ /dev/null @@ -1 +0,0 @@ -Fix build failure on systems without thread-locals support. diff --git a/Misc/NEWS.d/next/C_API/2024-07-03-13-39-13.gh-issue-121058.MKi1MV.rst b/Misc/NEWS.d/next/C_API/2024-07-03-13-39-13.gh-issue-121058.MKi1MV.rst deleted file mode 100644 index 133d8cb6fe4b9e..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-07-03-13-39-13.gh-issue-121058.MKi1MV.rst +++ /dev/null @@ -1,2 +0,0 @@ -``PyThreadState_Clear()`` now warns (and calls ``sys.excepthook``) if the -thread state still has an active exception. diff --git a/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst b/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst deleted file mode 100644 index c18c159ac87d08..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst +++ /dev/null @@ -1,10 +0,0 @@ -Add a new import and export API for Python :class:`int` objects (:pep:`757`): - -* :c:func:`PyLong_GetNativeLayout`; -* :c:func:`PyLong_Export`; -* :c:func:`PyLong_FreeExport`; -* :c:func:`PyLongWriter_Create`; -* :c:func:`PyLongWriter_Finish`; -* :c:func:`PyLongWriter_Discard`. - -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst b/Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst deleted file mode 100644 index 9d6b2e0c565623..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst +++ /dev/null @@ -1 +0,0 @@ -Reading text from a non-blocking stream with ``read`` may now raise a :exc:`BlockingIOError` if the operation cannot immediately return bytes. diff --git a/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst b/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst deleted file mode 100644 index 7254a04f61843d..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst +++ /dev/null @@ -1,6 +0,0 @@ -Ensure that the value of :attr:`UnicodeEncodeError.start ` -retrieved by :c:func:`PyUnicodeEncodeError_GetStart` lies in -``[0, max(0, objlen - 1)]`` where *objlen* is the length of -:attr:`UnicodeEncodeError.object `. Similar -arguments apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` -and their corresponding C interface. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst b/Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst deleted file mode 100644 index 8ea3c4ee2a2c53..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve error message when calling the C API without an active thread state -on the :term:`free-threaded ` build. diff --git a/Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst b/Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst deleted file mode 100644 index 107751579c4d91..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst +++ /dev/null @@ -1,6 +0,0 @@ -Ensure that the value of :attr:`UnicodeEncodeError.end ` -retrieved by :c:func:`PyUnicodeEncodeError_GetEnd` lies in ``[min(1, objlen), -max(min(1, objlen), objlen)]`` where *objlen* is the length of -:attr:`UnicodeEncodeError.object `. Similar arguments -apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` and their -corresponding C interface. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/C_API/2024-12-06-16-53-34.gh-issue-127691.k_Jitp.rst b/Misc/NEWS.d/next/C_API/2024-12-06-16-53-34.gh-issue-127691.k_Jitp.rst deleted file mode 100644 index c942ff3d9eda53..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-12-06-16-53-34.gh-issue-127691.k_Jitp.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :ref:`Unicode Exception Objects ` C API -now raises a :exc:`TypeError` if its exception argument is not -a :exc:`UnicodeError` object. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst b/Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst deleted file mode 100644 index 70751f18f5cf17..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix loss of callbacks after more than one call to -:c:func:`PyUnstable_AtExit`. diff --git a/Misc/NEWS.d/next/C_API/2024-12-16-07-12-15.gh-issue-127896.HmI9pk.rst b/Misc/NEWS.d/next/C_API/2024-12-16-07-12-15.gh-issue-127896.HmI9pk.rst deleted file mode 100644 index 82b4f563591fe1..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-12-16-07-12-15.gh-issue-127896.HmI9pk.rst +++ /dev/null @@ -1,2 +0,0 @@ -The previously undocumented function :c:func:`PySequence_In` is :term:`soft deprecated`. -Use :c:func:`PySequence_Contains` instead. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2023-09-22-21-01-56.gh-issue-109746.32MHt9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2023-09-22-21-01-56.gh-issue-109746.32MHt9.rst deleted file mode 100644 index 2d350c33aa6975..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2023-09-22-21-01-56.gh-issue-109746.32MHt9.rst +++ /dev/null @@ -1 +0,0 @@ -If :func:`!_thread.start_new_thread` fails to start a new thread, it deletes its state from interpreter and thus avoids its repeated cleanup on finalization. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst deleted file mode 100644 index 7954c7f5927397..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correct invalid corner cases which resulted in ``(nan+nanj)`` output in complex -multiplication, e.g., ``(1e300+1j)*(nan+infj)``. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-03-14-02-27.gh-issue-69639.mW3iKq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-03-14-02-27.gh-issue-69639.mW3iKq.rst deleted file mode 100644 index 72596b0302aa45..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-03-14-02-27.gh-issue-69639.mW3iKq.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement mixed-mode arithmetic rules combining real and complex numbers -as specified by C standards since C99. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-21-50-23.gh-issue-124470.pFr3_d.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-21-50-23.gh-issue-124470.pFr3_d.rst deleted file mode 100644 index 8f2f37146d3c13..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-21-50-23.gh-issue-124470.pFr3_d.rst +++ /dev/null @@ -1 +0,0 @@ -Fix crash in free-threaded builds when replacing object dictionary while reading attribute on another thread diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst deleted file mode 100644 index ef120802b5dc59..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :meth:`memoryview.count` to :class:`memoryview` objects. Patch by -Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst deleted file mode 100644 index 6ed823175f6754..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :meth:`memoryview.index` to :class:`memoryview` objects. Patch by -Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst deleted file mode 100644 index b41fff30433c34..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst +++ /dev/null @@ -1,2 +0,0 @@ -Optimize decoding of short UTF-8 sequences containing non-ASCII characters -by approximately 15%. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-07-21-48-23.gh-issue-126091.ETaRGE.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-07-21-48-23.gh-issue-126091.ETaRGE.rst deleted file mode 100644 index 08118ff1af657d..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-07-21-48-23.gh-issue-126091.ETaRGE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ensure stack traces are complete when throwing into a generator chain that -ends in a custom generator. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-15-16-39-37.gh-issue-126892.QR6Yo3.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-15-16-39-37.gh-issue-126892.QR6Yo3.rst deleted file mode 100644 index db3c398e5dbdbe..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-15-16-39-37.gh-issue-126892.QR6Yo3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Require cold or invalidated code to "warm up" before being JIT compiled -again. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst deleted file mode 100644 index 13381c7630d7ce..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix crash in finalization of dtoa state. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst deleted file mode 100644 index fd1570908c1fd6..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst +++ /dev/null @@ -1 +0,0 @@ -Increase performance of :class:`int` by adding a freelist for compact ints. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst deleted file mode 100644 index 8d7da0d4107021..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :exc:`TypeError` when a :class:`ctypes.Structure` has a field size -that doesn't fit into an unsigned 16-bit integer. -Instead, the maximum number of *bits* is :data:`sys.maxsize`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-23-18-17.gh-issue-126980.r8QHdi.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-23-18-17.gh-issue-126980.r8QHdi.rst deleted file mode 100644 index 84484e7c3001da..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-23-18-17.gh-issue-126980.r8QHdi.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :meth:`~object.__buffer__` of :class:`bytearray` crashing when -:attr:`~inspect.BufferFlags.READ` or :attr:`~inspect.BufferFlags.WRITE` are -passed as flags. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-19-17-17-32.gh-issue-127010.9Cl4bb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-19-17-17-32.gh-issue-127010.9Cl4bb.rst deleted file mode 100644 index 36e379c88ab27e..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-19-17-17-32.gh-issue-127010.9Cl4bb.rst +++ /dev/null @@ -1,4 +0,0 @@ -Simplify GC tracking of dictionaries. All dictionaries are tracked when -created, rather than being lazily tracked when a trackable object was added -to them. This simplifies the code considerably and results in a slight -speedup. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-19-21-49-58.gh-issue-127020.5vvI17.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-19-21-49-58.gh-issue-127020.5vvI17.rst deleted file mode 100644 index a8fd9272f5a923..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-19-21-49-58.gh-issue-127020.5vvI17.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a crash in the free threading build when :c:func:`PyCode_GetCode`, -:c:func:`PyCode_GetVarnames`, :c:func:`PyCode_GetCellvars`, or -:c:func:`PyCode_GetFreevars` were called from multiple threads at the same -time. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst deleted file mode 100644 index 9ef2b8dc33ed0f..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add a marking phase to the GC. All objects that can be transitively reached -from builtin modules or the stacks are marked as reachable before cycle -detection. This reduces the amount of work done by the GC by approximately -half. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-23-04-54-42.gh-issue-127133.WMoJjF.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-23-04-54-42.gh-issue-127133.WMoJjF.rst deleted file mode 100644 index 56b496bdf1e310..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-23-04-54-42.gh-issue-127133.WMoJjF.rst +++ /dev/null @@ -1,6 +0,0 @@ -Calling :meth:`argparse.ArgumentParser.add_argument_group` on an argument group, -and calling :meth:`argparse.ArgumentParser.add_argument_group` or -:meth:`argparse.ArgumentParser.add_mutually_exclusive_group` on a mutually -exclusive group now raise exceptions. This nesting was never supported, often -failed to work correctly, and was unintentionally exposed through inheritance. -This functionality has been deprecated since Python 3.11. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-24-07-01-28.gh-issue-113841.WFg-Bu.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-24-07-01-28.gh-issue-113841.WFg-Bu.rst deleted file mode 100644 index 2b07fdfcc6b527..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-24-07-01-28.gh-issue-113841.WFg-Bu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix possible undefined behavior division by zero in :class:`complex`'s -:c:func:`_Py_c_pow`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-25-05-15-21.gh-issue-127238.O8wkH-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-25-05-15-21.gh-issue-127238.O8wkH-.rst deleted file mode 100644 index e8a274fcd31f26..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-25-05-15-21.gh-issue-127238.O8wkH-.rst +++ /dev/null @@ -1 +0,0 @@ -Correct error message for :func:`sys.set_int_max_str_digits`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst deleted file mode 100644 index a59b9502eaa33e..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst +++ /dev/null @@ -1 +0,0 @@ -Fix race when exporting a buffer from a :class:`memoryview` object on the :term:`free-threaded ` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst deleted file mode 100644 index 6e2b87fe38146b..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add missing locks around some list assignment operations in the free -threading build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst deleted file mode 100644 index 08b27a7890bb1c..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst +++ /dev/null @@ -1 +0,0 @@ -The iOS compiler shims can now accept arguments with spaces. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst deleted file mode 100644 index 59491feeb9bcfa..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix non-thread-safe object resurrection when calling finalizers and watcher -callbacks in the free threading build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst deleted file mode 100644 index 92b18b082eb30e..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst +++ /dev/null @@ -1 +0,0 @@ -When raising :exc:`ImportError` for missing symbols in ``from`` imports, use ``__file__`` in the error message if ``__spec__.origin`` is not a location diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-07-13-06-09.gh-issue-127599.tXCZb_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-07-13-06-09.gh-issue-127599.tXCZb_.rst deleted file mode 100644 index 565ecb82c71926..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-07-13-06-09.gh-issue-127599.tXCZb_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix statistics for increments of object reference counts (in particular, when -a reference count was increased by more than 1 in a single operation). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst deleted file mode 100644 index 248e1b4855afb8..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -``PySequence_Tuple`` now creates the resulting tuple atomically, preventing -partially created tuples being visible to the garbage collector or through -``gc.get_referrers()`` diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst deleted file mode 100644 index f614dbb59bdc87..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix error message in :func:`bytes.fromhex` when given an odd number of -digits to properly indicate that an even number of hexadecimal digits is -required. diff --git a/Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst b/Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst deleted file mode 100644 index 79b3faa3d0d385..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst +++ /dev/null @@ -1 +0,0 @@ -Publicly expose :func:`traceback.print_list` in :attr:`!traceback.__all__`. diff --git a/Misc/NEWS.d/next/Library/2022-11-10-17-16-45.gh-issue-97514.kzA0zl.rst b/Misc/NEWS.d/next/Library/2022-11-10-17-16-45.gh-issue-97514.kzA0zl.rst deleted file mode 100644 index 10c56edb8c7303..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-10-17-16-45.gh-issue-97514.kzA0zl.rst +++ /dev/null @@ -1,10 +0,0 @@ -Authentication was added to the :mod:`multiprocessing` forkserver start -method control socket so that only processes with the authentication key -generated by the process that spawned the forkserver can control it. This -is an enhancement over the other :gh:`97514` fixes so that access is no -longer limited only by filesystem permissions. - -The file descriptor exchange of control pipes with the forked worker process -now requires an explicit acknowledgement byte to be sent over the socket after -the exchange on all forkserver supporting platforms. That makes testing the -above much easier. diff --git a/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst b/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst deleted file mode 100644 index 42a83edc3ba68d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed :class:`multiprocessing.Process` reporting a ``.exitcode`` of 1 even on success when -using the ``"fork"`` start method while using a :class:`concurrent.futures.ThreadPoolExecutor`. diff --git a/Misc/NEWS.d/next/Library/2024-07-25-18-06-51.gh-issue-122288.-_xxOR.rst b/Misc/NEWS.d/next/Library/2024-07-25-18-06-51.gh-issue-122288.-_xxOR.rst deleted file mode 100644 index 26a18afca945d9..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-07-25-18-06-51.gh-issue-122288.-_xxOR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve the performances of :func:`fnmatch.translate` by a factor 1.7. Patch -by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-07-29-15-20-30.gh-issue-122356.wKCmFx.rst b/Misc/NEWS.d/next/Library/2024-07-29-15-20-30.gh-issue-122356.wKCmFx.rst deleted file mode 100644 index 0a4632ca975f6b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-07-29-15-20-30.gh-issue-122356.wKCmFx.rst +++ /dev/null @@ -1,3 +0,0 @@ -Guarantee that the position of a file-like object passed to -:func:`zipfile.is_zipfile` is left untouched after the call. -Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst b/Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst deleted file mode 100644 index 16ad75792aefa2..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`readline.append_history_file` now raises a :exc:`ValueError` when given a negative value. diff --git a/Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst b/Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst deleted file mode 100644 index 638f3f76239ca6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :mod:`http.cookies` module now supports parsing obsolete :rfc:`850` -date formats, in accordance with :rfc:`9110` requirements. -Patch by Nano Zheng. diff --git a/Misc/NEWS.d/next/Library/2024-09-13-18-24-27.gh-issue-124008.XaiPQx.rst b/Misc/NEWS.d/next/Library/2024-09-13-18-24-27.gh-issue-124008.XaiPQx.rst deleted file mode 100644 index cd6dd9a7a97e90..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-09-13-18-24-27.gh-issue-124008.XaiPQx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix possible crash (in debug build), incorrect output or returning incorrect -value from raw binary ``write()`` when writing to console on Windows. diff --git a/Misc/NEWS.d/next/Library/2024-10-23-20-05-54.gh-issue-86463.jvFTI_.rst b/Misc/NEWS.d/next/Library/2024-10-23-20-05-54.gh-issue-86463.jvFTI_.rst deleted file mode 100644 index 9ac155770e2254..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-10-23-20-05-54.gh-issue-86463.jvFTI_.rst +++ /dev/null @@ -1,2 +0,0 @@ -The ``usage`` parameter of :class:`argparse.ArgumentParser` no longer -affects the default value of the ``prog`` parameter in subparsers. diff --git a/Misc/NEWS.d/next/Library/2024-10-28-19-49-18.gh-issue-118201.v41XXh.rst b/Misc/NEWS.d/next/Library/2024-10-28-19-49-18.gh-issue-118201.v41XXh.rst deleted file mode 100644 index bed4b3b5956f31..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-10-28-19-49-18.gh-issue-118201.v41XXh.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed intermittent failures of :any:`os.confstr`, :any:`os.pathconf` and -:any:`os.sysconf` on iOS and Android. diff --git a/Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst b/Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst deleted file mode 100644 index ae1392e2caf387..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst +++ /dev/null @@ -1 +0,0 @@ -Add :data:`~errno.EHWPOISON` error code to :mod:`errno`. diff --git a/Misc/NEWS.d/next/Library/2024-11-12-13-14-47.gh-issue-126727.5Eqfqd.rst b/Misc/NEWS.d/next/Library/2024-11-12-13-14-47.gh-issue-126727.5Eqfqd.rst deleted file mode 100644 index 7bec8a6b7a830a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-12-13-14-47.gh-issue-126727.5Eqfqd.rst +++ /dev/null @@ -1,3 +0,0 @@ -``locale.nl_langinfo(locale.ERA)`` now returns multiple era description -segments separated by semicolons. Previously it only returned the first -segment on platforms with Glibc. diff --git a/Misc/NEWS.d/next/Library/2024-11-12-20-05-09.gh-issue-126601.Nj7bA9.rst b/Misc/NEWS.d/next/Library/2024-11-12-20-05-09.gh-issue-126601.Nj7bA9.rst deleted file mode 100644 index 11e2b7350a0e48..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-12-20-05-09.gh-issue-126601.Nj7bA9.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix issue where :func:`urllib.request.pathname2url` raised :exc:`OSError` -when given a Windows path containing a colon character not following a -drive letter, such as before an NTFS alternate data stream. diff --git a/Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst b/Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst deleted file mode 100644 index 429fc2aa17bd42..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst +++ /dev/null @@ -1 +0,0 @@ -Make :func:`linecache.checkcache` thread safe and GC re-entrancy safe. diff --git a/Misc/NEWS.d/next/Library/2024-11-13-19-15-18.gh-issue-126780.ZZqJvI.rst b/Misc/NEWS.d/next/Library/2024-11-13-19-15-18.gh-issue-126780.ZZqJvI.rst deleted file mode 100644 index 93d45caf5cad72..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-13-19-15-18.gh-issue-126780.ZZqJvI.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`os.path.normpath` for drive-relative paths on Windows. diff --git a/Misc/NEWS.d/next/Library/2024-11-15-01-50-36.gh-issue-85168.bP8VIN.rst b/Misc/NEWS.d/next/Library/2024-11-15-01-50-36.gh-issue-85168.bP8VIN.rst deleted file mode 100644 index abceda8f6fd707..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-15-01-50-36.gh-issue-85168.bP8VIN.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix issue where :func:`urllib.request.url2pathname` and -:func:`~urllib.request.pathname2url` always used UTF-8 when quoting and -unquoting file URIs. They now use the :term:`filesystem encoding and error -handler`. diff --git a/Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst b/Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst deleted file mode 100644 index c1a0ed6438135d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make tkinter widget methods :meth:`!after` and :meth:`!after_idle` accept -arguments passed by keyword. diff --git a/Misc/NEWS.d/next/Library/2024-11-18-16-43-11.gh-issue-126946.52Ou-B.rst b/Misc/NEWS.d/next/Library/2024-11-18-16-43-11.gh-issue-126946.52Ou-B.rst deleted file mode 100644 index 448055ccfdff40..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-18-16-43-11.gh-issue-126946.52Ou-B.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve the :exc:`~getopt.GetoptError` error message when a long option -prefix matches multiple accepted options in :func:`getopt.getopt` and -:func:`getopt.gnu_getopt`. diff --git a/Misc/NEWS.d/next/Library/2024-11-18-19-03-46.gh-issue-126947.NiDYUe.rst b/Misc/NEWS.d/next/Library/2024-11-18-19-03-46.gh-issue-126947.NiDYUe.rst deleted file mode 100644 index 29ba4f21454fe1..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-18-19-03-46.gh-issue-126947.NiDYUe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Raise :exc:`TypeError` in :meth:`!_pydatetime.timedelta.__new__` if the passed arguments are not :class:`int` or :class:`float`, so that the Python -implementation is in line with the C implementation. diff --git a/Misc/NEWS.d/next/Library/2024-11-18-22-02-47.gh-issue-118761.GQKD_J.rst b/Misc/NEWS.d/next/Library/2024-11-18-22-02-47.gh-issue-118761.GQKD_J.rst deleted file mode 100644 index ebb9fe8016de21..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-18-22-02-47.gh-issue-118761.GQKD_J.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve import time of :mod:`mimetypes` by around 11-16 times. Patch by Hugo -van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst b/Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst deleted file mode 100644 index b169c1508d0d30..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst +++ /dev/null @@ -1 +0,0 @@ -In the :mod:`trace` module, increase the coverage precision (``cov%``) to one decimal. diff --git a/Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst b/Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst deleted file mode 100644 index c875c7b547bba9..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst +++ /dev/null @@ -1,3 +0,0 @@ -When running under a virtual environment with the :mod:`site` disabled (see -:option:`-S`), :data:`sys.prefix` and :data:`sys.base_prefix` will now point -to the virtual environment, instead of the base installation. diff --git a/Misc/NEWS.d/next/Library/2024-11-19-14-34-05.gh-issue-126615.LOskwi.rst b/Misc/NEWS.d/next/Library/2024-11-19-14-34-05.gh-issue-126615.LOskwi.rst deleted file mode 100644 index 8c7a2ade03c19e..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-19-14-34-05.gh-issue-126615.LOskwi.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :exc:`~ctypes.COMError` exception is now public. -Previously, this was private and only available in ``_ctypes``. diff --git a/Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst b/Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst deleted file mode 100644 index 7a0a7b7517b70d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the representation of :class:`itertools.count` objects when the count -value is :data:`sys.maxsize`. diff --git a/Misc/NEWS.d/next/Library/2024-11-20-11-37-08.gh-issue-126316.ElkZmE.rst b/Misc/NEWS.d/next/Library/2024-11-20-11-37-08.gh-issue-126316.ElkZmE.rst deleted file mode 100644 index d643254c5b3564..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-20-11-37-08.gh-issue-126316.ElkZmE.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`grp`: Make :func:`grp.getgrall` thread-safe by adding a mutex. Patch -by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-11-20-16-58-59.gh-issue-126997.0PI41Y.rst b/Misc/NEWS.d/next/Library/2024-11-20-16-58-59.gh-issue-126997.0PI41Y.rst deleted file mode 100644 index b85c51ef07dcbe..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-20-16-58-59.gh-issue-126997.0PI41Y.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix support of STRING and GLOBAL opcodes with non-ASCII arguments in -:mod:`pickletools`. :func:`pickletools.dis` now outputs non-ASCII bytes in -STRING, BINSTRING and SHORT_BINSTRING arguments as escaped (``\xXX``). diff --git a/Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst b/Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst deleted file mode 100644 index 526785f68cc807..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix LONG and INT opcodes to only use base 10 for string to integer conversion in :mod:`pickle`. diff --git a/Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst b/Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst deleted file mode 100644 index 8efe563f443774..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix value of :attr:`urllib.response.addinfourl.url` for ``file:`` URLs that -express relative paths and absolute Windows paths. The canonical URL generated -by :func:`urllib.request.pathname2url` is now used. diff --git a/Misc/NEWS.d/next/Library/2024-11-21-16-23-16.gh-issue-127065.cfL1zd.rst b/Misc/NEWS.d/next/Library/2024-11-21-16-23-16.gh-issue-127065.cfL1zd.rst deleted file mode 100644 index 83457da467ffa9..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-21-16-23-16.gh-issue-127065.cfL1zd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash when calling a :func:`operator.methodcaller` instance from -multiple threads in the free threading build. diff --git a/Misc/NEWS.d/next/Library/2024-11-22-02-31-55.gh-issue-126766.jfkhBH.rst b/Misc/NEWS.d/next/Library/2024-11-22-02-31-55.gh-issue-126766.jfkhBH.rst deleted file mode 100644 index 998c99bf4358d5..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-22-02-31-55.gh-issue-126766.jfkhBH.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where :func:`urllib.request.url2pathname` failed to discard any -'localhost' authority present in the URL. diff --git a/Misc/NEWS.d/next/Library/2024-11-22-03-40-02.gh-issue-127078.gI_PaP.rst b/Misc/NEWS.d/next/Library/2024-11-22-03-40-02.gh-issue-127078.gI_PaP.rst deleted file mode 100644 index a84c06f3c7a273..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-22-03-40-02.gh-issue-127078.gI_PaP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where :func:`urllib.request.url2pathname` failed to discard an -extra slash before a UNC drive in the URL path on Windows. diff --git a/Misc/NEWS.d/next/Library/2024-11-22-04-49-31.gh-issue-125866.TUtvPK.rst b/Misc/NEWS.d/next/Library/2024-11-22-04-49-31.gh-issue-125866.TUtvPK.rst deleted file mode 100644 index 682e061747689b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-22-04-49-31.gh-issue-125866.TUtvPK.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`urllib.request.pathname2url` and :func:`~urllib.request.url2pathname` -no longer convert Windows drive letters to uppercase. diff --git a/Misc/NEWS.d/next/Library/2024-11-22-09-23-41.gh-issue-122273.H8M6fd.rst b/Misc/NEWS.d/next/Library/2024-11-22-09-23-41.gh-issue-122273.H8M6fd.rst deleted file mode 100644 index 99071e05377e33..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-22-09-23-41.gh-issue-122273.H8M6fd.rst +++ /dev/null @@ -1 +0,0 @@ -Support PyREPL history on Windows. Patch by devdanzin and Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-11-22-10-42-34.gh-issue-127035.UnbDlr.rst b/Misc/NEWS.d/next/Library/2024-11-22-10-42-34.gh-issue-127035.UnbDlr.rst deleted file mode 100644 index 6bb7abfdd50040..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-22-10-42-34.gh-issue-127035.UnbDlr.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :mod:`shutil.which` on Windows. Now it looks at direct match if and only -if the command ends with a PATHEXT extension or X_OK is not in mode. Support -extensionless files if "." is in PATHEXT. Support PATHEXT extensions that end -with a dot. diff --git a/Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst b/Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst deleted file mode 100644 index 0e4a03caf9f49d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst +++ /dev/null @@ -1 +0,0 @@ -Add colour to :mod:`unittest` output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-11-23-12-25-06.gh-issue-125866.wEOP66.rst b/Misc/NEWS.d/next/Library/2024-11-23-12-25-06.gh-issue-125866.wEOP66.rst deleted file mode 100644 index 0b8ffdb3901db3..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-23-12-25-06.gh-issue-125866.wEOP66.rst +++ /dev/null @@ -1,5 +0,0 @@ -:func:`urllib.request.pathname2url` now adds an empty authority when -generating a URL for a path that begins with exactly one slash. For example, -the path ``/etc/hosts`` is converted to the scheme-less URL ``///etc/hosts``. -As a result of this change, URLs without authorities are only generated for -relative paths. diff --git a/Misc/NEWS.d/next/Library/2024-11-24-12-41-31.gh-issue-127217.UAXGFr.rst b/Misc/NEWS.d/next/Library/2024-11-24-12-41-31.gh-issue-127217.UAXGFr.rst deleted file mode 100644 index 3139e33302f378..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-24-12-41-31.gh-issue-127217.UAXGFr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`urllib.request.pathname2url` for paths starting with multiple -slashes on Posix. diff --git a/Misc/NEWS.d/next/Library/2024-11-24-14-20-17.gh-issue-127182.WmfY2g.rst b/Misc/NEWS.d/next/Library/2024-11-24-14-20-17.gh-issue-127182.WmfY2g.rst deleted file mode 100644 index 2cc46ca3d33977..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-24-14-20-17.gh-issue-127182.WmfY2g.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`!io.StringIO.__setstate__` crash, when :const:`None` was passed as -the first value. diff --git a/Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst b/Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst deleted file mode 100644 index 9fe7815e93cf4f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :func:`~ctypes.CopyComPointer` function is now public. -Previously, this was private and only available in ``_ctypes``. diff --git a/Misc/NEWS.d/next/Library/2024-11-25-19-04-10.gh-issue-127072.-c284K.rst b/Misc/NEWS.d/next/Library/2024-11-25-19-04-10.gh-issue-127072.-c284K.rst deleted file mode 100644 index 1bc7e1f0de9e0b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-25-19-04-10.gh-issue-127072.-c284K.rst +++ /dev/null @@ -1 +0,0 @@ -Remove outdated ``socket.NETLINK_*`` constants not present in Linux kernels beyond 2.6.17. diff --git a/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst b/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst deleted file mode 100644 index b703b58ea8e1d9..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst +++ /dev/null @@ -1,4 +0,0 @@ -A ``_sysconfig_vars_(...).json`` file is now shipped in the standard library -directory. It contains the output of :func:`sysconfig.get_config_vars` on -the default environment encoded as JSON data. This is an implementation -detail, and may change at any time. diff --git a/Misc/NEWS.d/next/Library/2024-11-27-14-06-35.gh-issue-123967.wxUmnW.rst b/Misc/NEWS.d/next/Library/2024-11-27-14-06-35.gh-issue-123967.wxUmnW.rst deleted file mode 100644 index 788fe0c78ef257..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-27-14-06-35.gh-issue-123967.wxUmnW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix faulthandler for trampoline frames. If the top-most frame is a -trampoline frame, skip it. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-11-27-14-23-02.gh-issue-127331.9sNEC9.rst b/Misc/NEWS.d/next/Library/2024-11-27-14-23-02.gh-issue-127331.9sNEC9.rst deleted file mode 100644 index c668816955ca59..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-27-14-23-02.gh-issue-127331.9sNEC9.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`ssl` can show descriptions for errors added in OpenSSL 3.4. diff --git a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst deleted file mode 100644 index 58ebf5d0abe141..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst +++ /dev/null @@ -1 +0,0 @@ -Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`. diff --git a/Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst b/Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst deleted file mode 100644 index a8c7b3d00755e6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Linux, :class:`threading.Thread` now sets the thread name to the -operating system. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst b/Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst deleted file mode 100644 index fb0380cba0b607..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst +++ /dev/null @@ -1,2 +0,0 @@ -In :mod:`ssl`, system call failures that OpenSSL reports using -``ERR_LIB_SYS`` are now raised as :exc:`OSError`. diff --git a/Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst b/Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst deleted file mode 100644 index b56a77b4294ace..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst +++ /dev/null @@ -1,3 +0,0 @@ -Revert addition of :meth:`!pathlib.Path.scandir`. This method was added in -3.14.0a2. The optimizations remain for file system paths, but other -subclasses should only have to implement :meth:`pathlib.Path.iterdir`. diff --git a/Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst b/Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst deleted file mode 100644 index 2330fb66253265..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add the :option:`dis --specialized` command-line option to show specialized -bytecode. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst deleted file mode 100644 index 708c1a6437d812..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed bug where, on cross-builds, the :mod:`sysconfig` POSIX data was being -generated with the host Python's ``Makefile``. The data is now generated from -current build's ``Makefile``. diff --git a/Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst b/Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst deleted file mode 100644 index 69b6ce68a47509..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`pdb.set_trace` will not stop at an opcode that does not have an associated line number anymore. diff --git a/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst b/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst deleted file mode 100644 index 03d6953b9ddfa1..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst +++ /dev/null @@ -1 +0,0 @@ -Make :func:`operator.methodcaller` thread-safe and re-entrant safe. diff --git a/Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst b/Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst deleted file mode 100644 index 8ada0b57ddc257..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst +++ /dev/null @@ -1 +0,0 @@ -Add the ``EPOLLWAKEUP`` constant to the :mod:`select` module. diff --git a/Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst b/Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst deleted file mode 100644 index e245fa2bdd00b4..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Include ```` to get ``os.PIDFD_NONBLOCK`` constant. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst b/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst deleted file mode 100644 index 677acf5baab3fa..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst +++ /dev/null @@ -1,2 +0,0 @@ -macOS and iOS apps can now choose to redirect stdout and stderr to the -system log during interpreter configuration. diff --git a/Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst b/Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst deleted file mode 100644 index 48a6c7d30b4a26..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added ``posix._emscripten_debugger()`` to help with debugging the test suite on -the Emscripten target. diff --git a/Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst b/Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst deleted file mode 100644 index 58769029d79977..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added validation for more than one var-positional or -var-keyword parameters in :class:`inspect.Signature`. -Patch by Maxim Ageev. diff --git a/Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst b/Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst deleted file mode 100644 index 6c1b7bef610e8c..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst +++ /dev/null @@ -1 +0,0 @@ -Add colour to :mod:`test.regrtest` output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst b/Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst deleted file mode 100644 index 417e9ac986f27a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fixed :func:`sysconfig.get_config_vars`, :func:`sysconfig.get_paths`, and -siblings, returning outdated cached data if the value of :data:`sys.prefix` -or :data:`sys.exec_prefix` changes. Overwriting :data:`sys.prefix` or -:data:`sys.exec_prefix` still is discouraged, as that might break other -parts of the code. diff --git a/Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst b/Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst deleted file mode 100644 index 44821300f6e4e6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst +++ /dev/null @@ -1 +0,0 @@ -The :mod:`platform` module now correctly detects Windows Server 2025. diff --git a/Misc/NEWS.d/next/Library/2024-12-12-16-59-42.gh-issue-127870._NFG-3.rst b/Misc/NEWS.d/next/Library/2024-12-12-16-59-42.gh-issue-127870._NFG-3.rst deleted file mode 100644 index 99b2df00032082..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-12-16-59-42.gh-issue-127870._NFG-3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Detect recursive calls in ctypes ``_as_parameter_`` handling. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-12-13-22-20-54.gh-issue-126907.fWRL_R.rst b/Misc/NEWS.d/next/Library/2024-12-13-22-20-54.gh-issue-126907.fWRL_R.rst deleted file mode 100644 index d33d2aa23b21b3..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-13-22-20-54.gh-issue-126907.fWRL_R.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash when using :mod:`atexit` concurrently on the :term:`free-threaded -` build. diff --git a/Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst b/Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst deleted file mode 100644 index 76cfc58121d3bd..00000000000000 --- a/Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed the :class:`!asyncio.selector_events._SelectorSocketTransport` transport not pausing writes for the protocol when the buffer reaches the high water mark when using :meth:`asyncio.WriteTransport.writelines`. diff --git a/Misc/NEWS.d/next/Tests/2024-11-20-18-49-01.gh-issue-127076.DHnXxo.rst b/Misc/NEWS.d/next/Tests/2024-11-20-18-49-01.gh-issue-127076.DHnXxo.rst deleted file mode 100644 index 39323604bbef56..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-11-20-18-49-01.gh-issue-127076.DHnXxo.rst +++ /dev/null @@ -1,2 +0,0 @@ -Filter out memory-related ``mmap``, ``munmap``, and ``mprotect`` calls from -file-related ones when testing :mod:`io` behavior using strace. diff --git a/Misc/NEWS.d/next/Tests/2024-11-21-02-03-48.gh-issue-127076.a3avV1.rst b/Misc/NEWS.d/next/Tests/2024-11-21-02-03-48.gh-issue-127076.a3avV1.rst deleted file mode 100644 index 7dec8bd627c063..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-11-21-02-03-48.gh-issue-127076.a3avV1.rst +++ /dev/null @@ -1 +0,0 @@ -Disable strace based system call tests when LD_PRELOAD is set. diff --git a/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst b/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst deleted file mode 100644 index fb307c7cb9bf1d..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst +++ /dev/null @@ -1,2 +0,0 @@ -iOS test results are now streamed during test execution, and the deprecated -xcresulttool is no longer used. diff --git a/Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst b/Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst deleted file mode 100644 index ac5d9827b07199..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst +++ /dev/null @@ -1 +0,0 @@ -Add tests for the :mod:`dis` command-line interface. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Tests/2024-12-13-13-41-34.gh-issue-127906.NuRHlB.rst b/Misc/NEWS.d/next/Tests/2024-12-13-13-41-34.gh-issue-127906.NuRHlB.rst deleted file mode 100644 index 6f577e741dff7f..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-12-13-13-41-34.gh-issue-127906.NuRHlB.rst +++ /dev/null @@ -1 +0,0 @@ -Test the limited C API in test_cppext. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-11-16-20-47-20.gh-issue-126700.ayrHv4.rst b/Misc/NEWS.d/next/Tools-Demos/2024-11-16-20-47-20.gh-issue-126700.ayrHv4.rst deleted file mode 100644 index c08ad9d7059904..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2024-11-16-20-47-20.gh-issue-126700.ayrHv4.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for multi-argument :mod:`gettext` functions in :program:`pygettext.py`. diff --git a/Misc/NEWS.d/next/Windows/2024-10-31-09-46-53.gh-issue-125729.KdKVLa.rst b/Misc/NEWS.d/next/Windows/2024-10-31-09-46-53.gh-issue-125729.KdKVLa.rst deleted file mode 100644 index fbf4ab1cd1a11a..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-10-31-09-46-53.gh-issue-125729.KdKVLa.rst +++ /dev/null @@ -1 +0,0 @@ -Makes the presence of the :mod:`turtle` module dependent on the Tcl/Tk installer option. Previously, the module was always installed but would be unusable without Tcl/Tk. diff --git a/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst b/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst deleted file mode 100644 index 88661b9a611071..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-11-28-15-55-48.gh-issue-127353.i-XOXg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow to force color output on Windows using environment variables. Patch by -Andrey Efremov. diff --git a/README.rst b/README.rst index 394cdc3638485d..02776205e6dcc9 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.14.0 alpha 2 +This is Python version 3.14.0 alpha 3 ===================================== .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push From 7303f06846b69016a075bca7ad7c6055f29ad024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:12:45 +0100 Subject: [PATCH 225/427] gh-126742: Add _PyErr_SetLocaleString, use it for gdbm & dlerror messages (GH-126746) - Add a helper to set an error from locale-encoded `char*` - Use the helper for gdbm & dlerror messages Co-authored-by: Victor Stinner --- Include/internal/pycore_pyerrors.h | 12 +++++ Lib/test/test_ctypes/test_dlerror.py | 80 ++++++++++++++++++++++++---- Lib/test/test_dbm_gnu.py | 21 ++++++-- Modules/_ctypes/_ctypes.c | 28 +++------- Modules/_ctypes/callproc.c | 38 +++++++------ Modules/_gdbmmodule.c | 44 +++++++++++---- Modules/_hashopenssl.c | 1 + Modules/_sqlite/util.c | 1 + Modules/_testcapi/exceptions.c | 1 + Modules/main.c | 1 + Modules/pyexpat.c | 7 ++- Python/errors.c | 9 ++++ 12 files changed, 175 insertions(+), 68 deletions(-) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 02945f0e71a145..6f2fdda9a9f12f 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -130,6 +130,18 @@ PyAPI_FUNC(void) _PyErr_SetString( PyObject *exception, const char *string); +/* + * Set an exception with the error message decoded from the current locale + * encoding (LC_CTYPE). + * + * Exceptions occurring in decoding take priority over the desired exception. + * + * Exported for '_ctypes' shared extensions. + */ +PyAPI_FUNC(void) _PyErr_SetLocaleString( + PyObject *exception, + const char *string); + PyAPI_FUNC(PyObject*) _PyErr_Format( PyThreadState *tstate, PyObject *exception, diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 4441e30cd7a2a7..c3c34d43481d36 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,7 +1,12 @@ +import _ctypes import os +import platform import sys +import test.support import unittest -import platform +from ctypes import CDLL, c_int +from ctypes.util import find_library + FOO_C = r""" #include @@ -26,7 +31,7 @@ @unittest.skipUnless(sys.platform.startswith('linux'), - 'Test only valid for Linux') + 'test requires GNU IFUNC support') class TestNullDlsym(unittest.TestCase): """GH-126554: Ensure that we catch NULL dlsym return values @@ -53,14 +58,6 @@ def test_null_dlsym(self): import subprocess import tempfile - # To avoid ImportErrors on Windows, where _ctypes does not have - # dlopen and dlsym, - # import here, i.e., inside the test function. - # The skipUnless('linux') decorator ensures that we're on linux - # if we're executing these statements. - from ctypes import CDLL, c_int - from _ctypes import dlopen, dlsym - retcode = subprocess.call(["gcc", "--version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -111,6 +108,8 @@ def test_null_dlsym(self): self.assertEqual(os.read(pipe_r, 2), b'OK') # Case #3: Test 'py_dl_sym' from Modules/_ctypes/callproc.c + dlopen = test.support.get_attribute(_ctypes, 'dlopen') + dlsym = test.support.get_attribute(_ctypes, 'dlsym') L = dlopen(dstname) with self.assertRaisesRegex(OSError, "symbol 'foo' not found"): dlsym(L, "foo") @@ -119,5 +118,66 @@ def test_null_dlsym(self): self.assertEqual(os.read(pipe_r, 2), b'OK') +@unittest.skipUnless(os.name != 'nt', 'test requires dlerror() calls') +class TestLocalization(unittest.TestCase): + + @staticmethod + def configure_locales(func): + return test.support.run_with_locale( + 'LC_ALL', + 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', + 'fr_FR.utf8', 'en_US.utf8', + '', + )(func) + + @classmethod + def setUpClass(cls): + cls.libc_filename = find_library("c") + + @configure_locales + def test_localized_error_from_dll(self): + dll = CDLL(self.libc_filename) + with self.assertRaises(AttributeError) as cm: + dll.this_name_does_not_exist + if sys.platform.startswith('linux'): + # On macOS, the filename is not reported by dlerror(). + self.assertIn(self.libc_filename, str(cm.exception)) + + @configure_locales + def test_localized_error_in_dll(self): + dll = CDLL(self.libc_filename) + with self.assertRaises(ValueError) as cm: + c_int.in_dll(dll, 'this_name_does_not_exist') + if sys.platform.startswith('linux'): + # On macOS, the filename is not reported by dlerror(). + self.assertIn(self.libc_filename, str(cm.exception)) + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') + @configure_locales + def test_localized_error_dlopen(self): + missing_filename = b'missing\xff.so' + # Depending whether the locale, we may encode '\xff' differently + # but we are only interested in avoiding a UnicodeDecodeError + # when reporting the dlerror() error message which contains + # the localized filename. + filename_pattern = r'missing.*?\.so' + with self.assertRaisesRegex(OSError, filename_pattern): + _ctypes.dlopen(missing_filename, 2) + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') + @unittest.skipUnless(hasattr(_ctypes, 'dlsym'), + 'test requires _ctypes.dlsym()') + @configure_locales + def test_localized_error_dlsym(self): + dll = _ctypes.dlopen(self.libc_filename) + with self.assertRaises(OSError) as cm: + _ctypes.dlsym(dll, 'this_name_does_not_exist') + if sys.platform.startswith('linux'): + # On macOS, the filename is not reported by dlerror(). + self.assertIn(self.libc_filename, str(cm.exception)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index e20addf1f04f1b..66268c42a300b5 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -1,10 +1,11 @@ -from test import support -from test.support import import_helper, cpython_only -gdbm = import_helper.import_module("dbm.gnu") #skip if not supported -import unittest import os -from test.support.os_helper import TESTFN, TESTFN_NONASCII, unlink, FakePath +import unittest +from test import support +from test.support import cpython_only, import_helper +from test.support.os_helper import (TESTFN, TESTFN_NONASCII, FakePath, + create_empty_file, temp_dir, unlink) +gdbm = import_helper.import_module("dbm.gnu") # skip if not supported filename = TESTFN @@ -205,6 +206,16 @@ def test_clear(self): self.assertNotIn(k, db) self.assertEqual(len(db), 0) + @support.run_with_locale( + 'LC_ALL', + 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', + 'fr_FR.utf8', 'en_US.utf8', + '', + ) + def test_localized_error(self): + with temp_dir() as d: + create_empty_file(os.path.join(d, 'test')) + self.assertRaises(gdbm.error, gdbm.open, filename, 'r') if __name__ == '__main__': diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index bb4699884057ba..3a3b1da5084a67 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -984,15 +984,8 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); - if (message) { - PyErr_SetObject(PyExc_ValueError, message); - Py_DECREF(message); - return NULL; - } - // Ignore errors from PyUnicode_DecodeLocale, - // fall back to the generic error below. - PyErr_Clear(); + _PyErr_SetLocaleString(PyExc_ValueError, dlerr); + return NULL; } #endif #undef USE_DLERROR @@ -3825,21 +3818,14 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) address = (PPROC)dlsym(handle, name); if (!address) { - #ifdef USE_DLERROR + #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); - if (message) { - PyErr_SetObject(PyExc_AttributeError, message); - Py_DECREF(ftuple); - Py_DECREF(message); - return NULL; - } - // Ignore errors from PyUnicode_DecodeLocale, - // fall back to the generic error below. - PyErr_Clear(); + _PyErr_SetLocaleString(PyExc_AttributeError, dlerr); + Py_DECREF(ftuple); + return NULL; } - #endif + #endif PyErr_Format(PyExc_AttributeError, "function '%s' not found", name); Py_DECREF(ftuple); return NULL; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 218c3a9c81e05f..92eedff5ec94f1 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1588,10 +1588,11 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) Py_XDECREF(name2); if (!handle) { const char *errmsg = dlerror(); - if (!errmsg) - errmsg = "dlopen() error"; - PyErr_SetString(PyExc_OSError, - errmsg); + if (errmsg) { + _PyErr_SetLocaleString(PyExc_OSError, errmsg); + return NULL; + } + PyErr_SetString(PyExc_OSError, "dlopen() error"); return NULL; } return PyLong_FromVoidPtr(handle); @@ -1604,8 +1605,12 @@ static PyObject *py_dl_close(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O&:dlclose", &_parse_voidp, &handle)) return NULL; if (dlclose(handle)) { - PyErr_SetString(PyExc_OSError, - dlerror()); + const char *errmsg = dlerror(); + if (errmsg) { + _PyErr_SetLocaleString(PyExc_OSError, errmsg); + return NULL; + } + PyErr_SetString(PyExc_OSError, "dlclose() error"); return NULL; } Py_RETURN_NONE; @@ -1639,21 +1644,14 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) if (ptr) { return PyLong_FromVoidPtr(ptr); } - #ifdef USE_DLERROR - const char *dlerr = dlerror(); - if (dlerr) { - PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); - if (message) { - PyErr_SetObject(PyExc_OSError, message); - Py_DECREF(message); - return NULL; - } - // Ignore errors from PyUnicode_DecodeLocale, - // fall back to the generic error below. - PyErr_Clear(); + #ifdef USE_DLERROR + const char *errmsg = dlerror(); + if (errmsg) { + _PyErr_SetLocaleString(PyExc_OSError, errmsg); + return NULL; } - #endif - #undef USE_DLERROR + #endif + #undef USE_DLERROR PyErr_Format(PyExc_OSError, "symbol '%s' not found", name); return NULL; } diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index df7fba67810ed0..ea4fe247987e9d 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -8,10 +8,11 @@ #endif #include "Python.h" +#include "pycore_pyerrors.h" // _PyErr_SetLocaleString() #include "gdbm.h" #include -#include // free() +#include // free() #include #include @@ -33,6 +34,24 @@ get_gdbm_state(PyObject *module) return (_gdbm_state *)state; } +/* + * Set the gdbm error obtained by gdbm_strerror(gdbm_errno). + * + * If no error message exists, a generic (UTF-8) error message + * is used instead. + */ +static void +set_gdbm_error(_gdbm_state *state, const char *generic_error) +{ + const char *gdbm_errmsg = gdbm_strerror(gdbm_errno); + if (gdbm_errmsg) { + _PyErr_SetLocaleString(state->gdbm_error, gdbm_errmsg); + } + else { + PyErr_SetString(state->gdbm_error, generic_error); + } +} + /*[clinic input] module _gdbm class _gdbm.gdbm "gdbmobject *" "&Gdbmtype" @@ -91,7 +110,7 @@ newgdbmobject(_gdbm_state *state, const char *file, int flags, int mode) PyErr_SetFromErrnoWithFilename(state->gdbm_error, file); } else { - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + set_gdbm_error(state, "gdbm_open() error"); } Py_DECREF(dp); return NULL; @@ -136,7 +155,7 @@ gdbm_length(gdbmobject *dp) PyErr_SetFromErrno(state->gdbm_error); } else { - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + set_gdbm_error(state, "gdbm_count() error"); } return -1; } @@ -286,7 +305,7 @@ gdbm_ass_sub(gdbmobject *dp, PyObject *v, PyObject *w) PyErr_SetObject(PyExc_KeyError, v); } else { - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + set_gdbm_error(state, "gdbm_delete() error"); } return -1; } @@ -297,11 +316,12 @@ gdbm_ass_sub(gdbmobject *dp, PyObject *v, PyObject *w) } errno = 0; if (gdbm_store(dp->di_dbm, krec, drec, GDBM_REPLACE) < 0) { - if (errno != 0) + if (errno != 0) { PyErr_SetFromErrno(state->gdbm_error); - else - PyErr_SetString(state->gdbm_error, - gdbm_strerror(gdbm_errno)); + } + else { + set_gdbm_error(state, "gdbm_store() error"); + } return -1; } } @@ -534,10 +554,12 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls) check_gdbmobject_open(self, state->gdbm_error); errno = 0; if (gdbm_reorganize(self->di_dbm) < 0) { - if (errno != 0) + if (errno != 0) { PyErr_SetFromErrno(state->gdbm_error); - else - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + } + else { + set_gdbm_error(state, "gdbm_reorganize() error"); + } return NULL; } Py_RETURN_NONE; diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 2c9a9feecc79f0..082929be3c77b7 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -319,6 +319,7 @@ _setException(PyObject *exc, const char* altmsg, ...) va_end(vargs); ERR_clear_error(); + /* ERR_ERROR_STRING(3) ensures that the messages below are ASCII */ lib = ERR_lib_error_string(errcode); func = ERR_func_error_string(errcode); reason = ERR_reason_error_string(errcode); diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 9e8613ef67916e..b0622e66928f47 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -134,6 +134,7 @@ _pysqlite_seterror(pysqlite_state *state, sqlite3 *db) /* Create and set the exception. */ int extended_errcode = sqlite3_extended_errcode(db); + // sqlite3_errmsg() always returns an UTF-8 encoded message const char *errmsg = sqlite3_errmsg(db); raise_exception(exc_class, extended_errcode, errmsg); return extended_errcode; diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index e92d9670e7c792..b647bfc71eae24 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -3,6 +3,7 @@ #include "parts.h" #include "util.h" + #include "clinic/exceptions.c.h" diff --git a/Modules/main.c b/Modules/main.c index 15ea49a1bad19e..3bf2241f2837a3 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -374,6 +374,7 @@ pymain_run_file_obj(PyObject *program_name, PyObject *filename, if (fp == NULL) { // Ignore the OSError PyErr_Clear(); + // TODO(picnixz): strerror() is locale dependent but not PySys_FormatStderr(). PySys_FormatStderr("%S: can't open file %R: [Errno %d] %s\n", program_name, filename, errno, strerror(errno)); return 2; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 9733bc34f7c80a..cf7714e7656205 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1782,7 +1782,12 @@ add_error(PyObject *errors_module, PyObject *codes_dict, * with the other uses of the XML_ErrorString function * elsewhere within this file. pyexpat's copy of the messages * only acts as a fallback in case of outdated runtime libexpat, - * where it returns NULL. */ + * where it returns NULL. + * + * In addition, XML_ErrorString is assumed to return UTF-8 encoded + * strings (in conv_string_to_unicode, we decode them using 'strict' + * error handling). + */ const char *error_string = XML_ErrorString(error_code); if (error_string == NULL) { error_string = error_info_of[error_index].description; diff --git a/Python/errors.c b/Python/errors.c index 7f3b4aabc432d7..2d362c1864ffff 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -301,6 +301,15 @@ PyErr_SetString(PyObject *exception, const char *string) _PyErr_SetString(tstate, exception, string); } +void +_PyErr_SetLocaleString(PyObject *exception, const char *string) +{ + PyObject *value = PyUnicode_DecodeLocale(string, "surrogateescape"); + if (value != NULL) { + PyErr_SetObject(exception, value); + Py_DECREF(value); + } +} PyObject* _Py_HOT_FUNCTION PyErr_Occurred(void) From d70e5c1fefb69c541205a7e97795f10fd61c2905 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:52:43 +0200 Subject: [PATCH 226/427] Post 3.14.0a3 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 1e9df2d9ebd07d..6d4f719fcde5a8 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 3 /* Version as a string */ -#define PY_VERSION "3.14.0a3" +#define PY_VERSION "3.14.0a3+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From b92f101d0f19a1df32050b8502cfcce777b079b2 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 18 Dec 2024 00:17:09 +0100 Subject: [PATCH 227/427] gh-127146: Emscripten Include compiler version in _PYTHON_HOST_PLATFORM (#127992) Modifies _PYTHON_HOST_PLATFORM to include the compiler version under Emscripten. The Emscripten compiler version is the platform version compatibility identifier. --- configure | 3 +++ configure.ac | 3 +++ 2 files changed, 6 insertions(+) diff --git a/configure b/configure index 57be576e3cae99..6df1116fc600f2 100755 --- a/configure +++ b/configure @@ -4545,6 +4545,9 @@ printf "%s\n" "$IPHONEOS_DEPLOYMENT_TARGET" >&6; } *-*-vxworks*) _host_ident=$host_cpu ;; + *-*-emscripten) + _host_ident=$(emcc -dumpversion)-$host_cpu + ;; wasm32-*-* | wasm64-*-*) _host_ident=$host_cpu ;; diff --git a/configure.ac b/configure.ac index bd0221481c5341..8295b59b8e45fb 100644 --- a/configure.ac +++ b/configure.ac @@ -793,6 +793,9 @@ if test "$cross_compiling" = yes; then *-*-vxworks*) _host_ident=$host_cpu ;; + *-*-emscripten) + _host_ident=$(emcc -dumpversion)-$host_cpu + ;; wasm32-*-* | wasm64-*-*) _host_ident=$host_cpu ;; From 559b0e7013f9cbf42a12fe9c86048d5cbb2f6f22 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Dec 2024 06:35:05 +0100 Subject: [PATCH 228/427] gh-127060: Disable traceback colors in IDLE (#128028) Set TERM environment variable to "dumb" to disable traceback colors in IDLE, since IDLE doesn't understand ANSI escape sequences. --- Lib/idlelib/pyshell.py | 4 +++- .../Library/2024-12-17-13-21-52.gh-issue-127060.mv2bX6.rst | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-17-13-21-52.gh-issue-127060.mv2bX6.rst diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index e882c6cb3b8d19..66fbbd4a97b7af 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -424,7 +424,9 @@ def __init__(self, tkconsole): def spawn_subprocess(self): if self.subprocess_arglist is None: self.subprocess_arglist = self.build_subprocess_arglist() - self.rpcsubproc = subprocess.Popen(self.subprocess_arglist) + # gh-127060: Disable traceback colors + env = dict(os.environ, TERM='dumb') + self.rpcsubproc = subprocess.Popen(self.subprocess_arglist, env=env) def build_subprocess_arglist(self): assert (self.port!=0), ( diff --git a/Misc/NEWS.d/next/Library/2024-12-17-13-21-52.gh-issue-127060.mv2bX6.rst b/Misc/NEWS.d/next/Library/2024-12-17-13-21-52.gh-issue-127060.mv2bX6.rst new file mode 100644 index 00000000000000..1da89e7a282147 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-17-13-21-52.gh-issue-127060.mv2bX6.rst @@ -0,0 +1,2 @@ +Set TERM environment variable to "dumb" to disable traceback colors in IDLE, +since IDLE doesn't understand ANSI escape sequences. Patch by Victor Stinner. From 5892853fb71acd6530e1e241a9a4bcf71a61fb21 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 18 Dec 2024 11:35:29 +0530 Subject: [PATCH 229/427] gh-127949: deprecate `asyncio.set_event_loop_policy` (#128024) First step towards deprecating the asyncio policy system. This deprecates `asyncio.set_event_loop_policy` and will be removed in Python 3.16. --- Doc/library/asyncio-policy.rst | 4 ++++ Lib/asyncio/events.py | 10 +++++++-- Lib/test/libregrtest/save_env.py | 2 +- Lib/test/test_asyncgen.py | 2 +- Lib/test/test_asyncio/test_base_events.py | 2 +- Lib/test/test_asyncio/test_buffered_proto.py | 2 +- Lib/test/test_asyncio/test_context.py | 2 +- .../test_asyncio/test_eager_task_factory.py | 2 +- Lib/test/test_asyncio/test_events.py | 22 +++++++++++-------- Lib/test/test_asyncio/test_futures.py | 2 +- Lib/test/test_asyncio/test_futures2.py | 2 +- Lib/test/test_asyncio/test_locks.py | 2 +- Lib/test/test_asyncio/test_pep492.py | 2 +- Lib/test/test_asyncio/test_proactor_events.py | 2 +- Lib/test/test_asyncio/test_protocols.py | 2 +- Lib/test/test_asyncio/test_queues.py | 2 +- Lib/test/test_asyncio/test_runners.py | 8 +++---- Lib/test/test_asyncio/test_selector_events.py | 2 +- Lib/test/test_asyncio/test_sendfile.py | 2 +- Lib/test/test_asyncio/test_server.py | 2 +- Lib/test/test_asyncio/test_sock_lowlevel.py | 2 +- Lib/test/test_asyncio/test_ssl.py | 2 +- Lib/test/test_asyncio/test_sslproto.py | 2 +- Lib/test/test_asyncio/test_staggered.py | 2 +- Lib/test/test_asyncio/test_streams.py | 2 +- Lib/test/test_asyncio/test_subprocess.py | 2 +- Lib/test/test_asyncio/test_taskgroups.py | 2 +- Lib/test/test_asyncio/test_tasks.py | 2 +- Lib/test/test_asyncio/test_threads.py | 2 +- Lib/test/test_asyncio/test_timeouts.py | 2 +- Lib/test/test_asyncio/test_transports.py | 2 +- Lib/test/test_asyncio/test_unix_events.py | 2 +- Lib/test/test_asyncio/test_waitfor.py | 2 +- Lib/test/test_asyncio/test_windows_events.py | 10 ++++----- Lib/test/test_asyncio/test_windows_utils.py | 2 +- Lib/test/test_builtin.py | 4 ++-- .../test_interpreter_pool.py | 2 +- Lib/test/test_contextlib_async.py | 2 +- Lib/test/test_coroutines.py | 2 +- Lib/test/test_inspect/test_inspect.py | 2 +- Lib/test/test_logging.py | 6 ++--- Lib/test/test_os.py | 2 +- Lib/test/test_pdb.py | 6 ++--- Lib/test/test_sys_settrace.py | 2 +- Lib/test/test_unittest/test_async_case.py | 4 ++-- Lib/test/test_unittest/testmock/testasync.py | 2 +- 46 files changed, 81 insertions(+), 67 deletions(-) diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 09b75762ff0272..e8e470c1b343fd 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -46,6 +46,10 @@ for the current process: If *policy* is set to ``None``, the default policy is restored. + .. deprecated:: next + The :func:`set_event_loop_policy` function is deprecated and + will be removed in Python 3.16. + .. _asyncio-policy-objects: diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index ca0a4f2fee5840..0926cfe2323478 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -8,7 +8,9 @@ 'AbstractEventLoopPolicy', 'AbstractEventLoop', 'AbstractServer', 'Handle', 'TimerHandle', - 'get_event_loop_policy', 'set_event_loop_policy', + 'get_event_loop_policy', + '_set_event_loop_policy', + 'set_event_loop_policy', 'get_event_loop', 'set_event_loop', 'new_event_loop', '_set_running_loop', 'get_running_loop', '_get_running_loop', @@ -21,6 +23,7 @@ import subprocess import sys import threading +import warnings from . import format_helpers @@ -765,7 +768,7 @@ def get_event_loop_policy(): return _event_loop_policy -def set_event_loop_policy(policy): +def _set_event_loop_policy(policy): """Set the current event loop policy. If policy is None, the default policy is restored.""" @@ -774,6 +777,9 @@ def set_event_loop_policy(policy): raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") _event_loop_policy = policy +def set_event_loop_policy(policy): + warnings._deprecated('set_event_loop_policy', remove=(3,16)) + _set_event_loop_policy(policy) def get_event_loop(): """Return an asyncio event loop. diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index b2cc381344b2ef..ffc29fa8dc686a 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -97,7 +97,7 @@ def get_asyncio_events__event_loop_policy(self): return support.maybe_get_event_loop_policy() def restore_asyncio_events__event_loop_policy(self, policy): asyncio = self.get_module('asyncio') - asyncio.set_event_loop_policy(policy) + asyncio._set_event_loop_policy(policy) def get_sys_argv(self): return id(sys.argv), sys.argv, sys.argv[:] diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 4f2278bb263681..4bce6d5c1b1d2f 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -629,7 +629,7 @@ def setUp(self): def tearDown(self): self.loop.close() self.loop = None - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def check_async_iterator_anext(self, ait_class): with self.subTest(anext="pure-Python"): diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index c14a0bb180d79b..08e38b047d519b 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -25,7 +25,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def mock_socket_module(): diff --git a/Lib/test/test_asyncio/test_buffered_proto.py b/Lib/test/test_asyncio/test_buffered_proto.py index f24e363ebfcfa3..9c386dd2e63815 100644 --- a/Lib/test/test_asyncio/test_buffered_proto.py +++ b/Lib/test/test_asyncio/test_buffered_proto.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class ReceiveStuffProto(asyncio.BufferedProtocol): diff --git a/Lib/test/test_asyncio/test_context.py b/Lib/test/test_asyncio/test_context.py index 6b80721873d95c..ad394f44e7e5f6 100644 --- a/Lib/test/test_asyncio/test_context.py +++ b/Lib/test/test_asyncio/test_context.py @@ -4,7 +4,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) @unittest.skipUnless(decimal.HAVE_CONTEXTVAR, "decimal is built with a thread-local context") diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 0e2b189f761521..dcf9ff716ad399 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -13,7 +13,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class EagerTaskFactoryLoopTests: diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 2ab638dc527aec..50df1b6ff9e09f 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -36,10 +36,10 @@ from test.support import socket_helper from test.support import threading_helper from test.support import ALWAYS_EQ, LARGEST, SMALLEST - +from test.support import warnings_helper def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def broken_unix_getsockname(): @@ -2764,13 +2764,17 @@ def test_get_event_loop_policy(self): self.assertIs(policy, asyncio.get_event_loop_policy()) def test_set_event_loop_policy(self): - self.assertRaises( - TypeError, asyncio.set_event_loop_policy, object()) + with self.assertWarnsRegex( + DeprecationWarning, "'set_event_loop_policy' is deprecated"): + self.assertRaises( + TypeError, asyncio.set_event_loop_policy, object()) old_policy = asyncio.get_event_loop_policy() policy = asyncio.DefaultEventLoopPolicy() - asyncio.set_event_loop_policy(policy) + with self.assertWarnsRegex( + DeprecationWarning, "'set_event_loop_policy' is deprecated"): + asyncio.set_event_loop_policy(policy) self.assertIs(policy, asyncio.get_event_loop_policy()) self.assertIsNot(policy, old_policy) @@ -2857,7 +2861,7 @@ def get_event_loop(self): old_policy = asyncio.get_event_loop_policy() try: - asyncio.set_event_loop_policy(Policy()) + asyncio._set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() with self.assertRaises(TestError): @@ -2885,7 +2889,7 @@ async def func(): asyncio.get_event_loop() finally: - asyncio.set_event_loop_policy(old_policy) + asyncio._set_event_loop_policy(old_policy) if loop is not None: loop.close() @@ -2897,7 +2901,7 @@ async def func(): def test_get_event_loop_returns_running_loop2(self): old_policy = asyncio.get_event_loop_policy() try: - asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) + asyncio._set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) loop = asyncio.new_event_loop() self.addCleanup(loop.close) @@ -2923,7 +2927,7 @@ async def func(): asyncio.get_event_loop() finally: - asyncio.set_event_loop_policy(old_policy) + asyncio._set_event_loop_policy(old_policy) if loop is not None: loop.close() diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 3a4291e3a68ca6..7db70a4c81d483 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -17,7 +17,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def _fakefunc(f): diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index b7cfffb76bd8f1..e2cddea01ecd93 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -7,7 +7,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class FutureTests: diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index c3bff760f7307e..aabfcd418829b2 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -20,7 +20,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class LockTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index 84c5f99129585b..48f4a75e0fd56c 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -11,7 +11,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) # Test that asyncio.iscoroutine() uses collections.abc.Coroutine diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index 4b3d551dd7b3a2..24c4e8546b17aa 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -18,7 +18,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def close_transport(transport): diff --git a/Lib/test/test_asyncio/test_protocols.py b/Lib/test/test_asyncio/test_protocols.py index 0f232631867db5..a8627b5b5b87f2 100644 --- a/Lib/test/test_asyncio/test_protocols.py +++ b/Lib/test/test_asyncio/test_protocols.py @@ -7,7 +7,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class ProtocolsAbsTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 5019e9a293525d..1a8d604faea1fd 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -6,7 +6,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class QueueBasicTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 45f70d09a2083a..f9afccc937f1de 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -12,7 +12,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def interrupt_self(): @@ -61,7 +61,7 @@ def setUp(self): super().setUp() policy = TestPolicy(self.new_loop) - asyncio.set_event_loop_policy(policy) + asyncio._set_event_loop_policy(policy) def tearDown(self): policy = asyncio.get_event_loop_policy() @@ -69,7 +69,7 @@ def tearDown(self): self.assertTrue(policy.loop.is_closed()) self.assertTrue(policy.loop.shutdown_ag_run) - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) super().tearDown() @@ -259,7 +259,7 @@ def new_event_loop(): loop.set_task_factory(Task) return loop - asyncio.set_event_loop_policy(TestPolicy(new_event_loop)) + asyncio._set_event_loop_policy(TestPolicy(new_event_loop)) with self.assertRaises(asyncio.CancelledError): asyncio.run(main()) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index efca30f37414f9..f984dc96415ba3 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -24,7 +24,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TestBaseSelectorEventLoop(BaseSelectorEventLoop): diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index 2509d4382cdebd..e1b766d06cbe1e 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -22,7 +22,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class MySendfileProto(asyncio.Protocol): diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 60a40cc8349fed..32211f4cba32cb 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -11,7 +11,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class BaseStartServer(func_tests.FunctionalTestCaseMixin): diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index acef24a703ba38..5b1e5143820cad 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -15,7 +15,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class MyProto(asyncio.Protocol): diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index e072ede29ee3c7..125a6c35793c44 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -29,7 +29,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class MyBaseProto(asyncio.Protocol): diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 761904c5146b6a..aa248c5786f634 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -21,7 +21,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) @unittest.skipIf(ssl is None, 'No ssl module') diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index 74941f704c4890..3c81b629693596 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class StaggeredTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index dbe5646c2b7c08..c3ba90b309e49f 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -21,7 +21,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class StreamTests(test_utils.TestCase): diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index ec748b9bb3e357..467e964b26c824 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -37,7 +37,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TestSubprocessTransport(base_subprocess.BaseSubprocessTransport): diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 1b4de96a572fb9..c47bf4ec9ed64b 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -14,7 +14,7 @@ # To prevent a warning "test altered the execution environment" def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class MyExc(Exception): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 9d2d356631b42c..5b8979a8bbd13a 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -24,7 +24,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) async def coroutine_function(): diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py index 774380270a7d70..c98c9a9b395ff9 100644 --- a/Lib/test/test_asyncio/test_threads.py +++ b/Lib/test/test_asyncio/test_threads.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class ToThreadTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index f5543e191d07ff..3ba84d63b2ca5f 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -9,7 +9,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TimeoutTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py index bbdb218efaa3b6..af10d3dc2a80df 100644 --- a/Lib/test/test_asyncio/test_transports.py +++ b/Lib/test/test_asyncio/test_transports.py @@ -10,7 +10,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TransportTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 021f45478d6f48..e9ee9702248015 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -33,7 +33,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) MOCK_ANY = mock.ANY diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index 11a8eeeab37634..d083f6b4d2a535 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) # The following value can be used as a very small timeout: diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 0c128c599ba011..26ca5f905f752f 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -19,7 +19,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class UpperProto(asyncio.Protocol): @@ -334,11 +334,11 @@ async def main(): old_policy = asyncio.get_event_loop_policy() try: - asyncio.set_event_loop_policy( + asyncio._set_event_loop_policy( asyncio.WindowsSelectorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio.set_event_loop_policy(old_policy) + asyncio._set_event_loop_policy(old_policy) def test_proactor_win_policy(self): async def main(): @@ -348,11 +348,11 @@ async def main(): old_policy = asyncio.get_event_loop_policy() try: - asyncio.set_event_loop_policy( + asyncio._set_event_loop_policy( asyncio.WindowsProactorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio.set_event_loop_policy(old_policy) + asyncio._set_event_loop_policy(old_policy) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index eafa5be3829682..be70720707cea7 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -16,7 +16,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class PipeTests(unittest.TestCase): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 06df217881a52f..a92edad86839e6 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -493,7 +493,7 @@ async def arange(n): asyncio.run(eval(co, globals_)) self.assertEqual(globals_['a'], 1) finally: - asyncio.set_event_loop_policy(policy) + asyncio._set_event_loop_policy(policy) def test_compile_top_level_await_invalid_cases(self): # helper function just to check we can run top=level async-for @@ -530,7 +530,7 @@ async def arange(n): mode, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) finally: - asyncio.set_event_loop_policy(policy) + asyncio._set_event_loop_policy(policy) def test_compile_async_generator(self): diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index ea1512fc830d0c..93eec08bfe10d5 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -311,7 +311,7 @@ def setUpClass(cls): # tests left a policy in place, just in case. policy = support.maybe_get_event_loop_policy() assert policy is None, policy - cls.addClassCleanup(lambda: asyncio.set_event_loop_policy(None)) + cls.addClassCleanup(lambda: asyncio._set_event_loop_policy(None)) def setUp(self): super().setUp() diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index ca7315783b9674..d8ee5757b12c15 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -11,7 +11,7 @@ support.requires_working_socket(module=True) def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TestAbstractAsyncContextManager(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index e6d65e7d90abb1..a72c43f9b47947 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2294,7 +2294,7 @@ async def f(): pass finally: loop.close() - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) self.assertEqual(buffer, [1, 2, 'MyException']) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 1ecf18bf49fa7e..2c950e46b3ed8a 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -75,7 +75,7 @@ def revise(filename, *args): def tearDownModule(): if support.has_socket_support: - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def signatures_with_lexicographic_keyword_only_parameters(): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 671b4c57a809aa..44c854f02a73c6 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5352,7 +5352,7 @@ def test_taskName_with_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) @support.requires_working_socket() def test_taskName_without_asyncio_imported(self): @@ -5364,7 +5364,7 @@ def test_taskName_without_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class BasicConfigTest(unittest.TestCase): @@ -5668,7 +5668,7 @@ async def log_record(): data = f.read().strip() self.assertRegex(data, r'Task-\d+ - hello world') finally: - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) if handler: handler.close() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index b0e686cb754b93..d688a225538c11 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -105,7 +105,7 @@ def create_file(filename, content=b'content'): def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class MiscTests(unittest.TestCase): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 48a4c568651879..58295cff84310f 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2132,7 +2132,7 @@ def test_pdb_next_command_for_asyncgen(): ... loop = asyncio.new_event_loop() ... loop.run_until_complete(test_main()) ... loop.close() - ... asyncio.set_event_loop_policy(None) + ... asyncio._set_event_loop_policy(None) ... print("finished") >>> with PdbTestInput(['step', @@ -2253,7 +2253,7 @@ def test_pdb_return_command_for_coroutine(): ... loop = asyncio.new_event_loop() ... loop.run_until_complete(test_main()) ... loop.close() - ... asyncio.set_event_loop_policy(None) + ... asyncio._set_event_loop_policy(None) ... print("finished") >>> with PdbTestInput(['step', @@ -2353,7 +2353,7 @@ def test_pdb_until_command_for_coroutine(): ... loop = asyncio.new_event_loop() ... loop.run_until_complete(test_main()) ... loop.close() - ... asyncio.set_event_loop_policy(None) + ... asyncio._set_event_loop_policy(None) ... print("finished") >>> with PdbTestInput(['step', diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 95cf0d1ec2d9ab..e5cf88177f7131 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2070,7 +2070,7 @@ def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, asyncio.run(func(output)) sys.settrace(None) - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) self.compare_jump_output(expected, output) def jump_test(jumpFrom, jumpTo, expected, error=None, event='line', warning=None): diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 8ea244bff05c5f..664ca5efe57f84 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -12,7 +12,7 @@ class MyException(Exception): def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TestCM: @@ -490,7 +490,7 @@ async def test_demo1(self): self.assertTrue(result.wasSuccessful()) def test_loop_factory(self): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TestCase1(unittest.IsolatedAsyncioTestCase): loop_factory = asyncio.EventLoop diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 73f04291373f91..afc9d1f11da1e2 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -15,7 +15,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class AsyncClass: From 329165639f9ac00ba64f6493dbcafcef6955e2cb Mon Sep 17 00:00:00 2001 From: aeiouaeiouaeiouaeiouaeiouaeiou Date: Wed, 18 Dec 2024 09:14:16 +0300 Subject: [PATCH 230/427] gh-127897: fix HACL* build on macOS/Catalina (GH-127932) gh-127897: Update HACL* module from upstream sources to get: - Lib_Memzero0.c: don't use memset_s() on macOS <10.9 - Use _mm_malloc() for KRML_ALIGNED_MALLOC on macOS <10.15 - Add LEGACY_MACOS macros, use _mm_free() for KRML_ALIGNED_FREE on macOS <10.15 --- Misc/sbom.spdx.json | 8 ++++---- Modules/_hacl/Lib_Memzero0.c | 6 +++++- Modules/_hacl/include/krml/internal/target.h | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 739e005646ba97..b4d785f65639a5 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -566,11 +566,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2e08072c0c57dac02b67f3f71d77068c537ac02e" + "checksumValue": "118dc712780ea680affa8d9794470440eb87ff10" }, { "algorithm": "SHA256", - "checksumValue": "e69fd3e84f77873ecb414f5300761b686321d01f5710ccf2517765236b08fc25" + "checksumValue": "b017e7d5662a308c938cf4e4b919680c8f3e27f42975ca152b62fe65c5f7fb0c" } ], "fileName": "Modules/_hacl/Lib_Memzero0.c" @@ -622,11 +622,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9881567f43deb32bae77a84b2d349858a24b6685" + "checksumValue": "9c5cac1582dcd6e0d0a4142e6e8b285b4cb7d9e6" }, { "algorithm": "SHA256", - "checksumValue": "3382156e32fcb376009177d3d2dc9712ff7c8c02afb97b3e16d98b41a2114f84" + "checksumValue": "b1e32138ac8c262e872f7da43ec80c1e54c08bcbdec4b7be17117aa25807f87e" } ], "fileName": "Modules/_hacl/include/krml/internal/target.h" diff --git a/Modules/_hacl/Lib_Memzero0.c b/Modules/_hacl/Lib_Memzero0.c index 5c269d231de82f..f01568a138648f 100644 --- a/Modules/_hacl/Lib_Memzero0.c +++ b/Modules/_hacl/Lib_Memzero0.c @@ -8,6 +8,10 @@ #include #endif +#if defined(__APPLE__) && defined(__MACH__) +#include +#endif + #if (defined(__APPLE__) && defined(__MACH__)) || defined(__linux__) #define __STDC_WANT_LIB_EXT1__ 1 #include @@ -37,7 +41,7 @@ void Lib_Memzero0_memzero0(void *dst, uint64_t len) { #ifdef _WIN32 SecureZeroMemory(dst, len_); - #elif defined(__APPLE__) && defined(__MACH__) + #elif defined(__APPLE__) && defined(__MACH__) && defined(MAC_OS_X_VERSION_MIN_REQUIRED) && (MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) memset_s(dst, len_, 0, len_); #elif (defined(__linux__) && !defined(LINUX_NO_EXPLICIT_BZERO)) || defined(__FreeBSD__) explicit_bzero(dst, len_); diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index fd74d3da684567..9b403c36ceca19 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -19,6 +19,20 @@ # define inline __inline__ #endif +/* There is no support for aligned_alloc() in macOS before Catalina, so + * let's make a macro to use _mm_malloc() and _mm_free() functions + * from mm_malloc.h. */ +#if defined(__APPLE__) && defined(__MACH__) +# include +# if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && \ + (MAC_OS_X_VERSION_MIN_REQUIRED < 101500) +# include +# define LEGACY_MACOS +# else +# undef LEGACY_MACOS +#endif +#endif + /******************************************************************************/ /* Macros that KaRaMeL will generate. */ /******************************************************************************/ @@ -133,6 +147,8 @@ defined(_MSC_VER) || \ (defined(__MINGW32__) && defined(__MINGW64_VERSION_MAJOR))) # define KRML_ALIGNED_MALLOC(X, Y) _aligned_malloc(Y, X) +# elif defined(LEGACY_MACOS) +# define KRML_ALIGNED_MALLOC(X, Y) _mm_malloc(Y, X) # else # define KRML_ALIGNED_MALLOC(X, Y) aligned_alloc(X, Y) # endif @@ -150,6 +166,8 @@ defined(_MSC_VER) || \ (defined(__MINGW32__) && defined(__MINGW64_VERSION_MAJOR))) # define KRML_ALIGNED_FREE(X) _aligned_free(X) +# elif defined(LEGACY_MACOS) +# define KRML_ALIGNED_FREE(X) _mm_free(X) # else # define KRML_ALIGNED_FREE(X) free(X) # endif From 0581e3f52b6116774698761088f0daa7ec23fc0b Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 18 Dec 2024 08:20:05 +0000 Subject: [PATCH 231/427] gh-127174: add docs for asyncio.get_event_loop replacements (#127640) Co-authored-by: Kumar Aditya --- Doc/whatsnew/3.14.rst | 90 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index d13cd2d5173a04..342456cbc397f3 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -780,6 +780,96 @@ asyncio It now raises a :exc:`RuntimeError` if there is no current event loop. (Contributed by Kumar Aditya in :gh:`126353`.) + There's a few patterns that use :func:`asyncio.get_event_loop`, most + of them can be replaced with :func:`asyncio.run`. + + If you're running an async function, simply use :func:`asyncio.run`. + + Before:: + + async def main(): + ... + + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(main()) + finally: + loop.close() + + After:: + + async def main(): + ... + + asyncio.run(main()) + + If you need to start something, e.g. a server listening on a socket + and then run forever, use :func:`asyncio.run` and an + :class:`asyncio.Event`. + + Before:: + + def start_server(loop): + ... + + loop = asyncio.get_event_loop() + try: + start_server(loop) + loop.run_forever() + finally: + loop.close() + + After:: + + def start_server(loop): + ... + + async def main(): + start_server(asyncio.get_running_loop()) + await asyncio.Event().wait() + + asyncio.run(main()) + + If you need to run something in an event loop, then run some blocking + code around it, use :class:`asyncio.Runner`. + + Before:: + + async def operation_one(): + ... + + def blocking_code(): + ... + + async def operation_two(): + ... + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(operation_one()) + blocking_code() + loop.run_until_complete(operation_two()) + finally: + loop.close() + + After:: + + async def operation_one(): + ... + + def blocking_code(): + ... + + async def operation_two(): + ... + + with asyncio.Runner() as runner: + runner.run(operation_one()) + blocking_code() + runner.run(operation_two()) + + collections.abc --------------- From 335e24fb0a5805ac1ecdbc01e0331b38fc33849d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:25:42 +0100 Subject: [PATCH 232/427] gh-126742: Avoid checking for library filename in test_ctypes (#128034) Avoid checking for library filename in `dlerror()` error messages of test_ctypes. --- Lib/test/test_ctypes/test_dlerror.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index c3c34d43481d36..6bf492399cbf95 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -133,24 +133,20 @@ def configure_locales(func): @classmethod def setUpClass(cls): cls.libc_filename = find_library("c") + if cls.libc_filename is None: + raise unittest.SkipTest('cannot find libc') @configure_locales def test_localized_error_from_dll(self): dll = CDLL(self.libc_filename) - with self.assertRaises(AttributeError) as cm: + with self.assertRaises(AttributeError): dll.this_name_does_not_exist - if sys.platform.startswith('linux'): - # On macOS, the filename is not reported by dlerror(). - self.assertIn(self.libc_filename, str(cm.exception)) @configure_locales def test_localized_error_in_dll(self): dll = CDLL(self.libc_filename) - with self.assertRaises(ValueError) as cm: + with self.assertRaises(ValueError): c_int.in_dll(dll, 'this_name_does_not_exist') - if sys.platform.startswith('linux'): - # On macOS, the filename is not reported by dlerror(). - self.assertIn(self.libc_filename, str(cm.exception)) @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), 'test requires _ctypes.dlopen()') @@ -172,11 +168,8 @@ def test_localized_error_dlopen(self): @configure_locales def test_localized_error_dlsym(self): dll = _ctypes.dlopen(self.libc_filename) - with self.assertRaises(OSError) as cm: + with self.assertRaises(OSError): _ctypes.dlsym(dll, 'this_name_does_not_exist') - if sys.platform.startswith('linux'): - # On macOS, the filename is not reported by dlerror(). - self.assertIn(self.libc_filename, str(cm.exception)) if __name__ == "__main__": From 2610bccfdf55bc6519808f8e1b5db2cfb03ae809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:59:37 +0100 Subject: [PATCH 233/427] gh-126742: add NEWS entry for fix of localized error messages (GH-128025) --- .../Library/2024-12-17-12-41-07.gh-issue-126742.l07qvT.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-12-17-12-41-07.gh-issue-126742.l07qvT.rst diff --git a/Misc/NEWS.d/next/Library/2024-12-17-12-41-07.gh-issue-126742.l07qvT.rst b/Misc/NEWS.d/next/Library/2024-12-17-12-41-07.gh-issue-126742.l07qvT.rst new file mode 100644 index 00000000000000..70f7cc129f66e3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-17-12-41-07.gh-issue-126742.l07qvT.rst @@ -0,0 +1,3 @@ +Fix support of localized error messages reported by :manpage:`dlerror(3)` and +:manpage:`gdbm_strerror ` in :mod:`ctypes` and :mod:`dbm.gnu` +functions respectively. Patch by Bénédikt Tran. From bad3cdefa840ff099e5e08cf88dcf6dfed7d37b8 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 18 Dec 2024 10:12:24 +0000 Subject: [PATCH 234/427] gh-126639: Add ResourceWarning to NamedTemporaryFile (#126677) Co-authored-by: Kumar Aditya --- Lib/tempfile.py | 26 ++++++++++++++++--- Lib/test/test_tempfile.py | 9 ++++--- ...-11-11-07-56-03.gh-issue-126639.AmVSt-.rst | 1 + 3 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-11-07-56-03.gh-issue-126639.AmVSt-.rst diff --git a/Lib/tempfile.py b/Lib/tempfile.py index b5a15f7b72c872..0eb9ddeb6ac377 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -437,11 +437,19 @@ class _TemporaryFileCloser: cleanup_called = False close_called = False - def __init__(self, file, name, delete=True, delete_on_close=True): + def __init__( + self, + file, + name, + delete=True, + delete_on_close=True, + warn_message="Implicitly cleaning up unknown file", + ): self.file = file self.name = name self.delete = delete self.delete_on_close = delete_on_close + self.warn_message = warn_message def cleanup(self, windows=(_os.name == 'nt'), unlink=_os.unlink): if not self.cleanup_called: @@ -469,7 +477,10 @@ def close(self): self.cleanup() def __del__(self): + close_called = self.close_called self.cleanup() + if not close_called: + _warnings.warn(self.warn_message, ResourceWarning) class _TemporaryFileWrapper: @@ -483,8 +494,17 @@ class _TemporaryFileWrapper: def __init__(self, file, name, delete=True, delete_on_close=True): self.file = file self.name = name - self._closer = _TemporaryFileCloser(file, name, delete, - delete_on_close) + self._closer = _TemporaryFileCloser( + file, + name, + delete, + delete_on_close, + warn_message=f"Implicitly cleaning up {self!r}", + ) + + def __repr__(self): + file = self.__dict__['file'] + return f"<{type(self).__name__} {file=}>" def __getattr__(self, name): # Attribute lookups are delegated to the underlying file diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 57e9bd20c77ee1..7adc021d298254 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1112,11 +1112,14 @@ def my_func(dir): # Testing extreme case, where the file is not explicitly closed # f.close() return tmp_name - # Make sure that the garbage collector has finalized the file object. - gc.collect() dir = tempfile.mkdtemp() try: - tmp_name = my_func(dir) + with self.assertWarnsRegex( + expected_warning=ResourceWarning, + expected_regex=r"Implicitly cleaning up <_TemporaryFileWrapper file=.*>", + ): + tmp_name = my_func(dir) + support.gc_collect() self.assertFalse(os.path.exists(tmp_name), f"NamedTemporaryFile {tmp_name!r} " f"exists after finalizer ") diff --git a/Misc/NEWS.d/next/Library/2024-11-11-07-56-03.gh-issue-126639.AmVSt-.rst b/Misc/NEWS.d/next/Library/2024-11-11-07-56-03.gh-issue-126639.AmVSt-.rst new file mode 100644 index 00000000000000..0b75e5858de731 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-11-07-56-03.gh-issue-126639.AmVSt-.rst @@ -0,0 +1 @@ +:class:`tempfile.NamedTemporaryFile` will now issue a :exc:`ResourceWarning` when it is finalized by the garbage collector without being explicitly closed. From dbd08fb60d1c434ebb6009d28d2b97f90e011032 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 18 Dec 2024 18:04:20 +0530 Subject: [PATCH 235/427] gh-127949: deprecate `asyncio.get_event_loop_policy` (#128053) This deprecates `asyncio.get_event_loop_policy` and will be removed in Python 3.16. --- Doc/library/asyncio-policy.rst | 4 +++ Lib/asyncio/events.py | 14 ++++++---- Lib/test/test_asyncio/test_events.py | 29 ++++++++++++-------- Lib/test/test_asyncio/test_runners.py | 6 ++-- Lib/test/test_asyncio/test_subprocess.py | 3 +- Lib/test/test_asyncio/test_windows_events.py | 4 +-- Lib/test/test_contextlib_async.py | 2 +- Lib/test/test_unittest/test_async_case.py | 2 +- Modules/_asynciomodule.c | 2 +- 9 files changed, 40 insertions(+), 26 deletions(-) diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index e8e470c1b343fd..2d05c3a9f7f157 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -40,6 +40,10 @@ for the current process: Return the current process-wide policy. + .. deprecated:: next + The :func:`get_event_loop_policy` function is deprecated and + will be removed in Python 3.16. + .. function:: set_event_loop_policy(policy) Set the current process-wide policy to *policy*. diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 0926cfe2323478..1449245edc7c7e 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -8,6 +8,7 @@ 'AbstractEventLoopPolicy', 'AbstractEventLoop', 'AbstractServer', 'Handle', 'TimerHandle', + '_get_event_loop_policy', 'get_event_loop_policy', '_set_event_loop_policy', 'set_event_loop_policy', @@ -761,12 +762,15 @@ def _init_event_loop_policy(): _event_loop_policy = DefaultEventLoopPolicy() -def get_event_loop_policy(): +def _get_event_loop_policy(): """Get the current event loop policy.""" if _event_loop_policy is None: _init_event_loop_policy() return _event_loop_policy +def get_event_loop_policy(): + warnings._deprecated('asyncio.get_event_loop_policy', remove=(3, 16)) + return _get_event_loop_policy() def _set_event_loop_policy(policy): """Set the current event loop policy. @@ -778,7 +782,7 @@ def _set_event_loop_policy(policy): _event_loop_policy = policy def set_event_loop_policy(policy): - warnings._deprecated('set_event_loop_policy', remove=(3,16)) + warnings._deprecated('asyncio.set_event_loop_policy', remove=(3,16)) _set_event_loop_policy(policy) def get_event_loop(): @@ -794,17 +798,17 @@ def get_event_loop(): current_loop = _get_running_loop() if current_loop is not None: return current_loop - return get_event_loop_policy().get_event_loop() + return _get_event_loop_policy().get_event_loop() def set_event_loop(loop): """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - get_event_loop_policy().set_event_loop(loop) + _get_event_loop_policy().set_event_loop(loop) def new_event_loop(): """Equivalent to calling get_event_loop_policy().new_event_loop().""" - return get_event_loop_policy().new_event_loop() + return _get_event_loop_policy().new_event_loop() # Alias pure-Python implementations for testing purposes. diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 50df1b6ff9e09f..d43f66c13d2f96 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2397,7 +2397,7 @@ def test_handle_repr_debug(self): self.assertRegex(repr(h), regex) def test_handle_source_traceback(self): - loop = asyncio.get_event_loop_policy().new_event_loop() + loop = asyncio.new_event_loop() loop.set_debug(True) self.set_event_loop(loop) @@ -2759,24 +2759,31 @@ def test_set_event_loop(self): old_loop.close() def test_get_event_loop_policy(self): - policy = asyncio.get_event_loop_policy() - self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) - self.assertIs(policy, asyncio.get_event_loop_policy()) + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + policy = asyncio.get_event_loop_policy() + self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) + self.assertIs(policy, asyncio.get_event_loop_policy()) def test_set_event_loop_policy(self): with self.assertWarnsRegex( - DeprecationWarning, "'set_event_loop_policy' is deprecated"): + DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): self.assertRaises( TypeError, asyncio.set_event_loop_policy, object()) - old_policy = asyncio.get_event_loop_policy() + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + old_policy = asyncio.get_event_loop_policy() policy = asyncio.DefaultEventLoopPolicy() with self.assertWarnsRegex( - DeprecationWarning, "'set_event_loop_policy' is deprecated"): + DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): asyncio.set_event_loop_policy(policy) - self.assertIs(policy, asyncio.get_event_loop_policy()) - self.assertIsNot(policy, old_policy) + + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + self.assertIs(policy, asyncio.get_event_loop_policy()) + self.assertIsNot(policy, old_policy) class GetEventLoopTestsMixin: @@ -2859,7 +2866,7 @@ class Policy(asyncio.DefaultEventLoopPolicy): def get_event_loop(self): raise TestError - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio._get_event_loop_policy() try: asyncio._set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() @@ -2899,7 +2906,7 @@ async def func(): self.assertIs(asyncio._get_running_loop(), None) def test_get_event_loop_returns_running_loop2(self): - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio._get_event_loop_policy() try: asyncio._set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) loop = asyncio.new_event_loop() diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index f9afccc937f1de..e1f82f7f7bec0c 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -64,7 +64,7 @@ def setUp(self): asyncio._set_event_loop_policy(policy) def tearDown(self): - policy = asyncio.get_event_loop_policy() + policy = asyncio._get_event_loop_policy() if policy.loop is not None: self.assertTrue(policy.loop.is_closed()) self.assertTrue(policy.loop.shutdown_ag_run) @@ -208,7 +208,7 @@ async def main(): await asyncio.sleep(0) return 42 - policy = asyncio.get_event_loop_policy() + policy = asyncio._get_event_loop_policy() policy.set_event_loop = mock.Mock() asyncio.run(main()) self.assertTrue(policy.set_event_loop.called) @@ -495,7 +495,7 @@ def test_set_event_loop_called_once(self): async def coro(): pass - policy = asyncio.get_event_loop_policy() + policy = asyncio._get_event_loop_policy() policy.set_event_loop = mock.Mock() runner = asyncio.Runner() runner.run(coro()) diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 467e964b26c824..57decaf2d277fb 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -886,8 +886,7 @@ class SubprocessWatcherMixin(SubprocessMixin): def setUp(self): super().setUp() - policy = asyncio.get_event_loop_policy() - self.loop = policy.new_event_loop() + self.loop = asyncio.new_event_loop() self.set_event_loop(self.loop) def test_watcher_implementation(self): diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 26ca5f905f752f..28b05d24dc25a1 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -332,7 +332,7 @@ async def main(): asyncio.get_running_loop(), asyncio.SelectorEventLoop) - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio._get_event_loop_policy() try: asyncio._set_event_loop_policy( asyncio.WindowsSelectorEventLoopPolicy()) @@ -346,7 +346,7 @@ async def main(): asyncio.get_running_loop(), asyncio.ProactorEventLoop) - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio._get_event_loop_policy() try: asyncio._set_event_loop_policy( asyncio.WindowsProactorEventLoopPolicy()) diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index d8ee5757b12c15..88dcdadd5e027d 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -496,7 +496,7 @@ class TestAsyncExitStack(TestBaseExitStack, unittest.IsolatedAsyncioTestCase): class SyncAsyncExitStack(AsyncExitStack): @staticmethod def run_coroutine(coro): - loop = asyncio.get_event_loop_policy().get_event_loop() + loop = asyncio.new_event_loop() t = loop.create_task(coro) t.add_done_callback(lambda f: loop.stop()) loop.run_forever() diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 664ca5efe57f84..993e6bf013cfbf 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -480,7 +480,7 @@ def test_setup_get_event_loop(self): class TestCase1(unittest.IsolatedAsyncioTestCase): def setUp(self): - asyncio.get_event_loop_policy().get_event_loop() + asyncio._get_event_loop_policy().get_event_loop() async def test_demo1(self): pass diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index f883125a2c70b2..13dd2fd55fd454 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3773,7 +3773,7 @@ module_init(asyncio_state *state) } WITH_MOD("asyncio.events") - GET_MOD_ATTR(state->asyncio_get_event_loop_policy, "get_event_loop_policy") + GET_MOD_ATTR(state->asyncio_get_event_loop_policy, "_get_event_loop_policy") WITH_MOD("asyncio.base_futures") GET_MOD_ATTR(state->asyncio_future_repr_func, "_future_repr") From 8a433b683fecafe1cb04469a301df2b4618167d0 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 18 Dec 2024 19:55:03 +0530 Subject: [PATCH 236/427] gh-121621: clear running loop early in asyncio (#128004) --- Modules/_asynciomodule.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 13dd2fd55fd454..27c16364457336 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3723,6 +3723,11 @@ module_clear(PyObject *mod) Py_CLEAR(state->iscoroutine_typecache); Py_CLEAR(state->context_kwname); + // Clear the ref to running loop so that finalizers can run early. + // If there are other running loops in different threads, + // those get cleared in PyThreadState_Clear. + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + Py_CLEAR(ts->asyncio_running_loop); return 0; } From 91c55085a959016250f1877e147ef379bb97dd12 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 18 Dec 2024 20:49:00 +0530 Subject: [PATCH 237/427] gh-128033: change `PyMutex_LockFast` to take `PyMutex` as argument (#128054) Change `PyMutex_LockFast` to take `PyMutex` as argument. --- Include/internal/pycore_critical_section.h | 6 +++--- Include/internal/pycore_lock.h | 3 ++- Python/ceval_macros.h | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 78cd0d54972660..9ba2fce56d3c9c 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -109,7 +109,7 @@ _PyCriticalSection_IsActive(uintptr_t tag) static inline void _PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) { - if (PyMutex_LockFast(&m->_bits)) { + if (PyMutex_LockFast(m)) { PyThreadState *tstate = _PyThreadState_GET(); c->_cs_mutex = m; c->_cs_prev = tstate->critical_section; @@ -170,8 +170,8 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) m2 = tmp; } - if (PyMutex_LockFast(&m1->_bits)) { - if (PyMutex_LockFast(&m2->_bits)) { + if (PyMutex_LockFast(m1)) { + if (PyMutex_LockFast(m2)) { PyThreadState *tstate = _PyThreadState_GET(); c->_cs_base._cs_mutex = m1; c->_cs_mutex2 = m2; diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 57cbce8f126aca..8bcb23a6ce9f9d 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -18,9 +18,10 @@ extern "C" { #define _Py_ONCE_INITIALIZED 4 static inline int -PyMutex_LockFast(uint8_t *lock_bits) +PyMutex_LockFast(PyMutex *m) { uint8_t expected = _Py_UNLOCKED; + uint8_t *lock_bits = &m->_bits; return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 9250b86e42ced1..398816d5f36a1d 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -300,7 +300,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { // avoid any potentially escaping calls (like PyStackRef_CLOSE) while the // object is locked. #ifdef Py_GIL_DISABLED -# define LOCK_OBJECT(op) PyMutex_LockFast(&(_PyObject_CAST(op))->ob_mutex._bits) +# define LOCK_OBJECT(op) PyMutex_LockFast(&(_PyObject_CAST(op))->ob_mutex) # define UNLOCK_OBJECT(op) PyMutex_Unlock(&(_PyObject_CAST(op))->ob_mutex) #else # define LOCK_OBJECT(op) (1) From f802c8bf872ab882d3056675acc79c950fe5b93c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Dec 2024 16:34:31 +0100 Subject: [PATCH 238/427] gh-128013: Convert unicodeobject.c macros to functions (#128061) Convert unicodeobject.c macros to static inline functions. * Add _PyUnicode_SET_UTF8() and _PyUnicode_SET_UTF8_LENGTH() macros. * Add PyUnicode_HASH() and PyUnicode_SET_HASH() macros. * Remove unused _PyUnicode_KIND() and _PyUnicode_GET_LENGTH() macros. --- Objects/unicodeobject.c | 123 +++++++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 45 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index b7aeb06d32bcec..53be6f5b9017a3 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -112,20 +112,42 @@ NOTE: In the interpreter's initialization phase, some globals are currently # define _PyUnicode_CHECK(op) PyUnicode_Check(op) #endif -#define _PyUnicode_UTF8(op) \ - (_PyCompactUnicodeObject_CAST(op)->utf8) -#define PyUnicode_UTF8(op) \ - (assert(_PyUnicode_CHECK(op)), \ - PyUnicode_IS_COMPACT_ASCII(op) ? \ - ((char*)(_PyASCIIObject_CAST(op) + 1)) : \ - _PyUnicode_UTF8(op)) -#define _PyUnicode_UTF8_LENGTH(op) \ - (_PyCompactUnicodeObject_CAST(op)->utf8_length) -#define PyUnicode_UTF8_LENGTH(op) \ - (assert(_PyUnicode_CHECK(op)), \ - PyUnicode_IS_COMPACT_ASCII(op) ? \ - _PyASCIIObject_CAST(op)->length : \ - _PyUnicode_UTF8_LENGTH(op)) +static inline char* _PyUnicode_UTF8(PyObject *op) +{ + return (_PyCompactUnicodeObject_CAST(op)->utf8); +} + +static inline char* PyUnicode_UTF8(PyObject *op) +{ + assert(_PyUnicode_CHECK(op)); + if (PyUnicode_IS_COMPACT_ASCII(op)) { + return ((char*)(_PyASCIIObject_CAST(op) + 1)); + } + else { + return _PyUnicode_UTF8(op); + } +} + +static inline void PyUnicode_SET_UTF8(PyObject *op, char *utf8) +{ + _PyCompactUnicodeObject_CAST(op)->utf8 = utf8; +} + +static inline Py_ssize_t PyUnicode_UTF8_LENGTH(PyObject *op) +{ + assert(_PyUnicode_CHECK(op)); + if (PyUnicode_IS_COMPACT_ASCII(op)) { + return _PyASCIIObject_CAST(op)->length; + } + else { + return _PyCompactUnicodeObject_CAST(op)->utf8_length; + } +} + +static inline void PyUnicode_SET_UTF8_LENGTH(PyObject *op, Py_ssize_t length) +{ + _PyCompactUnicodeObject_CAST(op)->utf8_length = length; +} #define _PyUnicode_LENGTH(op) \ (_PyASCIIObject_CAST(op)->length) @@ -133,26 +155,37 @@ NOTE: In the interpreter's initialization phase, some globals are currently (_PyASCIIObject_CAST(op)->state) #define _PyUnicode_HASH(op) \ (_PyASCIIObject_CAST(op)->hash) -#define _PyUnicode_KIND(op) \ - (assert(_PyUnicode_CHECK(op)), \ - _PyASCIIObject_CAST(op)->state.kind) -#define _PyUnicode_GET_LENGTH(op) \ - (assert(_PyUnicode_CHECK(op)), \ - _PyASCIIObject_CAST(op)->length) + +static inline Py_hash_t PyUnicode_HASH(PyObject *op) +{ + assert(_PyUnicode_CHECK(op)); + return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(op)->hash); +} + +static inline void PyUnicode_SET_HASH(PyObject *op, Py_hash_t hash) +{ + FT_ATOMIC_STORE_SSIZE_RELAXED(_PyASCIIObject_CAST(op)->hash, hash); +} + #define _PyUnicode_DATA_ANY(op) \ (_PyUnicodeObject_CAST(op)->data.any) -#define _PyUnicode_SHARE_UTF8(op) \ - (assert(_PyUnicode_CHECK(op)), \ - assert(!PyUnicode_IS_COMPACT_ASCII(op)), \ - (_PyUnicode_UTF8(op) == PyUnicode_DATA(op))) +static inline int _PyUnicode_SHARE_UTF8(PyObject *op) +{ + assert(_PyUnicode_CHECK(op)); + assert(!PyUnicode_IS_COMPACT_ASCII(op)); + return (_PyUnicode_UTF8(op) == PyUnicode_DATA(op)); +} /* true if the Unicode object has an allocated UTF-8 memory block (not shared with other data) */ -#define _PyUnicode_HAS_UTF8_MEMORY(op) \ - ((!PyUnicode_IS_COMPACT_ASCII(op) \ - && _PyUnicode_UTF8(op) \ - && _PyUnicode_UTF8(op) != PyUnicode_DATA(op))) +static inline int _PyUnicode_HAS_UTF8_MEMORY(PyObject *op) +{ + return (!PyUnicode_IS_COMPACT_ASCII(op) + && _PyUnicode_UTF8(op) != NULL + && _PyUnicode_UTF8(op) != PyUnicode_DATA(op)); +} + /* Generic helper macro to convert characters of different types. from_type and to_type have to be valid type names, begin and end @@ -1123,8 +1156,8 @@ resize_compact(PyObject *unicode, Py_ssize_t length) if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyMem_Free(_PyUnicode_UTF8(unicode)); - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; + PyUnicode_SET_UTF8(unicode, NULL); + PyUnicode_SET_UTF8_LENGTH(unicode, 0); } #ifdef Py_TRACE_REFS _Py_ForgetReference(unicode); @@ -1177,8 +1210,8 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyMem_Free(_PyUnicode_UTF8(unicode)); - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; + PyUnicode_SET_UTF8(unicode, NULL); + PyUnicode_SET_UTF8_LENGTH(unicode, 0); } data = (PyObject *)PyObject_Realloc(data, new_size); @@ -1188,8 +1221,8 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) } _PyUnicode_DATA_ANY(unicode) = data; if (share_utf8) { - _PyUnicode_UTF8(unicode) = data; - _PyUnicode_UTF8_LENGTH(unicode) = length; + PyUnicode_SET_UTF8(unicode, data); + PyUnicode_SET_UTF8_LENGTH(unicode, length); } _PyUnicode_LENGTH(unicode) = length; PyUnicode_WRITE(PyUnicode_KIND(unicode), data, length, 0); @@ -1769,7 +1802,7 @@ unicode_modifiable(PyObject *unicode) assert(_PyUnicode_CHECK(unicode)); if (Py_REFCNT(unicode) != 1) return 0; - if (FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(unicode)) != -1) + if (PyUnicode_HASH(unicode) != -1) return 0; if (PyUnicode_CHECK_INTERNED(unicode)) return 0; @@ -5862,8 +5895,8 @@ unicode_fill_utf8(PyObject *unicode) PyErr_NoMemory(); return -1; } - _PyUnicode_UTF8(unicode) = cache; - _PyUnicode_UTF8_LENGTH(unicode) = len; + PyUnicode_SET_UTF8(unicode, cache); + PyUnicode_SET_UTF8_LENGTH(unicode, len); memcpy(cache, start, len); cache[len] = '\0'; _PyBytesWriter_Dealloc(&writer); @@ -11434,9 +11467,9 @@ _PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) return 0; } - Py_hash_t right_hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(right_uni)); + Py_hash_t right_hash = PyUnicode_HASH(right_uni); assert(right_hash != -1); - Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(left)); + Py_hash_t hash = PyUnicode_HASH(left); if (hash != -1 && hash != right_hash) { return 0; } @@ -11916,14 +11949,14 @@ unicode_hash(PyObject *self) #ifdef Py_DEBUG assert(_Py_HashSecret_Initialized); #endif - Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(self)); + Py_hash_t hash = PyUnicode_HASH(self); if (hash != -1) { return hash; } x = Py_HashBuffer(PyUnicode_DATA(self), PyUnicode_GET_LENGTH(self) * PyUnicode_KIND(self)); - FT_ATOMIC_STORE_SSIZE_RELAXED(_PyUnicode_HASH(self), x); + PyUnicode_SET_HASH(self, x); return x; } @@ -15427,8 +15460,8 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) _PyUnicode_STATE(self).compact = 0; _PyUnicode_STATE(self).ascii = _PyUnicode_STATE(unicode).ascii; _PyUnicode_STATE(self).statically_allocated = 0; - _PyUnicode_UTF8_LENGTH(self) = 0; - _PyUnicode_UTF8(self) = NULL; + PyUnicode_SET_UTF8_LENGTH(self, 0); + PyUnicode_SET_UTF8(self, NULL); _PyUnicode_DATA_ANY(self) = NULL; share_utf8 = 0; @@ -15458,8 +15491,8 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) _PyUnicode_DATA_ANY(self) = data; if (share_utf8) { - _PyUnicode_UTF8_LENGTH(self) = length; - _PyUnicode_UTF8(self) = data; + PyUnicode_SET_UTF8_LENGTH(self, length); + PyUnicode_SET_UTF8(self, data); } memcpy(data, PyUnicode_DATA(unicode), kind * (length + 1)); From 48c70b8f7dfd00a018abbac50ea987f54fa4db51 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Thu, 19 Dec 2024 11:08:17 +0900 Subject: [PATCH 239/427] gh-115999: Enable BINARY_SUBSCR_GETITEM for free-threaded build (gh-127737) --- Include/internal/pycore_opcode_metadata.h | 4 +-- Include/internal/pycore_typeobject.h | 1 + Include/internal/pycore_uop_metadata.h | 2 +- Lib/test/test_opcache.py | 19 +++++++++++- Objects/typeobject.c | 25 ++++++++++++++++ Programs/test_frozenmain.h | 2 +- Python/bytecodes.c | 22 +++++++------- Python/executor_cases.c.h | 32 ++++++++++++--------- Python/generated_cases.c.h | 19 ++++++------ Python/optimizer_bytecodes.c | 3 +- Python/optimizer_cases.c.h | 16 ++++++++--- Python/specialize.c | 35 ++++++++++++----------- 12 files changed, 118 insertions(+), 62 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 28aa1120414337..d2ae8928a8fe8f 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -994,7 +994,7 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { return 0; } case BINARY_SUBSCR: { - *effect = 0; + *effect = 1; return 0; } case BINARY_SUBSCR_DICT: { @@ -1002,7 +1002,7 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { return 0; } case BINARY_SUBSCR_GETITEM: { - *effect = 0; + *effect = 1; return 0; } case BINARY_SUBSCR_LIST_INT: { diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 7b39d07f976ee3..581153344a8e05 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -278,6 +278,7 @@ typedef int (*_py_validate_type)(PyTypeObject *); // and if the validation is passed, it will set the ``tp_version`` as valid // tp_version_tag from the ``ty``. extern int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); +extern int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index dd775d3f7d3cdd..eadfda472a7270 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -722,7 +722,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _BINARY_SUBSCR_CHECK_FUNC: return 0; case _BINARY_SUBSCR_INIT_CALL: - return 2; + return 3; case _LIST_APPEND: return 1; case _SET_ADD: diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 0a7557adc4763b..94709e2022550a 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1069,7 +1069,7 @@ def write(items): opname = "STORE_SUBSCR_LIST_INT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_unpack_sequence_list(self): def get_items(): items = [] @@ -1245,6 +1245,14 @@ def f(o, n): f(test_obj, 1) self.assertEqual(test_obj.b, 0) +# gh-127274: BINARY_SUBSCR_GETITEM will only cache __getitem__ methods that +# are deferred. We only defer functions defined at the top-level. +class CGetItem: + def __init__(self, val): + self.val = val + def __getitem__(self, item): + return self.val + class TestSpecializer(TestBase): @@ -1520,6 +1528,15 @@ def binary_subscr_str_int(): self.assert_specialized(binary_subscr_str_int, "BINARY_SUBSCR_STR_INT") self.assert_no_opcode(binary_subscr_str_int, "BINARY_SUBSCR") + def binary_subscr_getitems(): + items = [CGetItem(i) for i in range(100)] + for i in range(100): + self.assertEqual(items[i][i], i) + + binary_subscr_getitems() + self.assert_specialized(binary_subscr_getitems, "BINARY_SUBSCR_GETITEM") + self.assert_no_opcode(binary_subscr_getitems, "BINARY_SUBSCR") + if __name__ == "__main__": unittest.main() diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2068d6aa9be52b..7f95b519561e68 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5679,6 +5679,31 @@ _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init, return can_cache; } +int +_PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version) +{ + if (!descriptor || !tp_version) { + return 0; + } + int can_cache; + BEGIN_TYPE_LOCK(); + can_cache = ((PyTypeObject*)ht)->tp_version_tag == tp_version; + // This pointer is invalidated by PyType_Modified (see the comment on + // struct _specialization_cache): + PyFunctionObject *func = (PyFunctionObject *)descriptor; + uint32_t version = _PyFunction_GetVersionForCurrentState(func); + can_cache = can_cache && _PyFunction_IsVersionValid(version); +#ifdef Py_GIL_DISABLED + can_cache = can_cache && _PyObject_HasDeferredRefcount(descriptor); +#endif + if (can_cache) { + FT_ATOMIC_STORE_PTR_RELEASE(ht->_spec_cache.getitem, descriptor); + FT_ATOMIC_STORE_UINT32_RELAXED(ht->_spec_cache.getitem_version, version); + } + END_TYPE_LOCK(); + return can_cache; +} + static void set_flags(PyTypeObject *self, unsigned long mask, unsigned long flags) { diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index c936622c020e3c..99b0fa48e01c8b 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,6 +1,6 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { - 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, + 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, 0,0,0,0,0,243,168,0,0,0,149,0,89,0,79,0, 70,0,111,0,89,0,79,0,70,1,111,1,88,2,31,0, 79,1,49,1,0,0,0,0,0,0,29,0,88,2,31,0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 772b46d17ec198..b67264f0440869 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -865,26 +865,24 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_BINARY_SUBSCR_CHECK_FUNC, (container, unused -- container, unused)) { + op(_BINARY_SUBSCR_CHECK_FUNC, (container, unused -- container, unused, getitem)) { PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - DEOPT_IF(getitem == NULL); - assert(PyFunction_Check(getitem)); - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(((PyFunctionObject *)getitem)->func_version != cached_version); - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem); + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + DEOPT_IF(getitem_o == NULL); + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); + getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } - op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _PyInterpreterFrame* )) { - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); + op(_BINARY_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _PyInterpreterFrame* )) { + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; INPUTS_DEAD(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 55e9c3aa2db64d..de61a64a6e3374 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1125,6 +1125,7 @@ case _BINARY_SUBSCR_CHECK_FUNC: { _PyStackRef container; + _PyStackRef getitem; container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { @@ -1132,42 +1133,45 @@ JUMP_TO_JUMP_TARGET(); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - if (getitem == NULL) { + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyFunction_Check(getitem)); - uint32_t cached_version = ht->_spec_cache.getitem_version; - if (((PyFunctionObject *)getitem)->func_version != cached_version) { + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); + stack_pointer[0] = getitem; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_INIT_CALL: { + _PyStackRef getitem; _PyStackRef sub; _PyStackRef container; _PyInterpreterFrame *new_frame; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); + getitem = stack_pointer[-1]; + sub = stack_pointer[-2]; + container = stack_pointer[-3]; + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; frame->return_offset = 2 ; - stack_pointer[-2].bits = (uintptr_t)new_frame; - stack_pointer += -1; + stack_pointer[-3].bits = (uintptr_t)new_frame; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 94343f953221eb..8a89ba890fd9c9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -505,6 +505,7 @@ INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); _PyStackRef container; + _PyStackRef getitem; _PyStackRef sub; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -518,23 +519,21 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - DEOPT_IF(getitem == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(getitem)); - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(((PyFunctionObject *)getitem)->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem); + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } // _BINARY_SUBSCR_INIT_CALL { sub = stack_pointer[-1]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; frame->return_offset = 2 ; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 0b8aff02367e31..e60c0d38425bfe 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -349,9 +349,10 @@ dummy_func(void) { GETLOCAL(this_instr->operand0) = res; } - op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_BINARY_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _Py_UOpsAbstractFrame *)) { (void)container; (void)sub; + (void)getitem; new_frame = NULL; ctx->done = true; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f4fbe8c8aa0480..33b34d6fa0d3f9 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -592,21 +592,29 @@ } case _BINARY_SUBSCR_CHECK_FUNC: { + _Py_UopsSymbol *getitem; + getitem = sym_new_not_null(ctx); + stack_pointer[0] = getitem; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_INIT_CALL: { + _Py_UopsSymbol *getitem; _Py_UopsSymbol *sub; _Py_UopsSymbol *container; _Py_UOpsAbstractFrame *new_frame; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; + getitem = stack_pointer[-1]; + sub = stack_pointer[-2]; + container = stack_pointer[-3]; (void)container; (void)sub; + (void)getitem; new_frame = NULL; ctx->done = true; - stack_pointer[-2] = (_Py_UopsSymbol *)new_frame; - stack_pointer += -1; + stack_pointer[-3] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/specialize.c b/Python/specialize.c index 6eb298217ec2d3..6c45320f95db8e 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1096,6 +1096,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; @@ -1165,6 +1166,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (version == 0) { return -1; } + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; @@ -1781,12 +1783,12 @@ _Py_Specialize_BinarySubscr( specialized_op = BINARY_SUBSCR_DICT; goto success; } -#ifndef Py_GIL_DISABLED - PyTypeObject *cls = Py_TYPE(container); - PyObject *descriptor = _PyType_Lookup(cls, &_Py_ID(__getitem__)); + unsigned int tp_version; + PyObject *descriptor = _PyType_LookupRefAndVersion(container_type, &_Py_ID(__getitem__), &tp_version); if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { if (!(container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE); + Py_DECREF(descriptor); goto fail; } PyFunctionObject *func = (PyFunctionObject *)descriptor; @@ -1794,30 +1796,29 @@ _Py_Specialize_BinarySubscr( int kind = function_kind(fcode); if (kind != SIMPLE_FUNCTION) { SPECIALIZATION_FAIL(BINARY_SUBSCR, kind); + Py_DECREF(descriptor); goto fail; } if (fcode->co_argcount != 2) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + Py_DECREF(descriptor); goto fail; } - uint32_t version = _PyFunction_GetVersionForCurrentState(func); - if (!_PyFunction_IsVersionValid(version)) { - SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS); - goto fail; - } + + PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type; + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OTHER); + Py_DECREF(descriptor); goto fail; } - PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type; - // This pointer is invalidated by PyType_Modified (see the comment on - // struct _specialization_cache): - ht->_spec_cache.getitem = descriptor; - ht->_spec_cache.getitem_version = version; - specialized_op = BINARY_SUBSCR_GETITEM; - goto success; + if (_PyType_CacheGetItemForSpecialization(ht, descriptor, (uint32_t)tp_version)) { + specialized_op = BINARY_SUBSCR_GETITEM; + Py_DECREF(descriptor); + goto success; + } } -#endif // Py_GIL_DISABLED + Py_XDECREF(descriptor); SPECIALIZATION_FAIL(BINARY_SUBSCR, binary_subscr_fail_kind(container_type, sub)); fail: @@ -2617,6 +2618,7 @@ _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR ); + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(FOR_ITER, SPEC_FAIL_OTHER); goto failure; @@ -2645,6 +2647,7 @@ _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND); PyTypeObject *tp = Py_TYPE(receiver); if (tp == &PyGen_Type || tp == &PyCoro_Type) { + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(SEND, SPEC_FAIL_OTHER); goto failure; From 46dc1ba9c6e8b95635fa27607d01d6108d8f677e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 19 Dec 2024 11:27:29 +0100 Subject: [PATCH 240/427] gh-128069: brew link --overwrite tcl-tk@8 to prevent conflict with GitHub image's version (#128090) brew link --overwrite tcl-tk@8 to prevent conflict with GitHub image's version Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/reusable-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 36ae3e27207e37..6fa389b2d66e5e 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -42,7 +42,7 @@ jobs: run: | brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make # Because alternate versions are not symlinked into place by default: - brew link tcl-tk@8 + brew link --overwrite tcl-tk@8 - name: Configure CPython run: | GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ From 3c168f7f79d1da2323d35dcf88c2d3c8730e5df6 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 19 Dec 2024 17:08:32 +0530 Subject: [PATCH 241/427] gh-128013: fix data race in `PyUnicode_AsUTF8AndSize` on free-threading (#128021) --- Lib/test/test_capi/test_unicode.py | 20 +++++++++++- Objects/unicodeobject.c | 49 +++++++++++++++++++----------- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 65d8242ad3fc60..3408c10f426058 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1,7 +1,7 @@ import unittest import sys from test import support -from test.support import import_helper +from test.support import threading_helper try: import _testcapi @@ -1005,6 +1005,24 @@ def test_asutf8(self): self.assertRaises(TypeError, unicode_asutf8, [], 0) # CRASHES unicode_asutf8(NULL, 0) + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @threading_helper.requires_working_threading() + def test_asutf8_race(self): + """Test that there's no race condition in PyUnicode_AsUTF8()""" + unicode_asutf8 = _testcapi.unicode_asutf8 + from threading import Thread + + data = "😊" + + def worker(): + for _ in range(1000): + self.assertEqual(unicode_asutf8(data, 5), b'\xf0\x9f\x98\x8a\0') + + threads = [Thread(target=worker) for _ in range(10)] + with threading_helper.start_threads(threads): + pass + + @support.cpython_only @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_asutf8andsize(self): diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 53be6f5b9017a3..1aab9cf37768a8 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -114,7 +114,7 @@ NOTE: In the interpreter's initialization phase, some globals are currently static inline char* _PyUnicode_UTF8(PyObject *op) { - return (_PyCompactUnicodeObject_CAST(op)->utf8); + return FT_ATOMIC_LOAD_PTR_ACQUIRE(_PyCompactUnicodeObject_CAST(op)->utf8); } static inline char* PyUnicode_UTF8(PyObject *op) @@ -130,7 +130,7 @@ static inline char* PyUnicode_UTF8(PyObject *op) static inline void PyUnicode_SET_UTF8(PyObject *op, char *utf8) { - _PyCompactUnicodeObject_CAST(op)->utf8 = utf8; + FT_ATOMIC_STORE_PTR_RELEASE(_PyCompactUnicodeObject_CAST(op)->utf8, utf8); } static inline Py_ssize_t PyUnicode_UTF8_LENGTH(PyObject *op) @@ -700,16 +700,17 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) CHECK(ascii->state.compact == 0); CHECK(data != NULL); if (ascii->state.ascii) { - CHECK(compact->utf8 == data); + CHECK(_PyUnicode_UTF8(op) == data); CHECK(compact->utf8_length == ascii->length); } else { - CHECK(compact->utf8 != data); + CHECK(_PyUnicode_UTF8(op) != data); } } - - if (compact->utf8 == NULL) +#ifndef Py_GIL_DISABLED + if (_PyUnicode_UTF8(op) == NULL) CHECK(compact->utf8_length == 0); +#endif } /* check that the best kind is used: O(n) operation */ @@ -1156,8 +1157,8 @@ resize_compact(PyObject *unicode, Py_ssize_t length) if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyMem_Free(_PyUnicode_UTF8(unicode)); - PyUnicode_SET_UTF8(unicode, NULL); PyUnicode_SET_UTF8_LENGTH(unicode, 0); + PyUnicode_SET_UTF8(unicode, NULL); } #ifdef Py_TRACE_REFS _Py_ForgetReference(unicode); @@ -1210,8 +1211,8 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyMem_Free(_PyUnicode_UTF8(unicode)); - PyUnicode_SET_UTF8(unicode, NULL); PyUnicode_SET_UTF8_LENGTH(unicode, 0); + PyUnicode_SET_UTF8(unicode, NULL); } data = (PyObject *)PyObject_Realloc(data, new_size); @@ -1221,8 +1222,8 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) } _PyUnicode_DATA_ANY(unicode) = data; if (share_utf8) { - PyUnicode_SET_UTF8(unicode, data); PyUnicode_SET_UTF8_LENGTH(unicode, length); + PyUnicode_SET_UTF8(unicode, data); } _PyUnicode_LENGTH(unicode) = length; PyUnicode_WRITE(PyUnicode_KIND(unicode), data, length, 0); @@ -4216,6 +4217,21 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) static int unicode_fill_utf8(PyObject *unicode); + +static int +unicode_ensure_utf8(PyObject *unicode) +{ + int err = 0; + if (PyUnicode_UTF8(unicode) == NULL) { + Py_BEGIN_CRITICAL_SECTION(unicode); + if (PyUnicode_UTF8(unicode) == NULL) { + err = unicode_fill_utf8(unicode); + } + Py_END_CRITICAL_SECTION(); + } + return err; +} + const char * PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { @@ -4227,13 +4243,11 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) return NULL; } - if (PyUnicode_UTF8(unicode) == NULL) { - if (unicode_fill_utf8(unicode) == -1) { - if (psize) { - *psize = -1; - } - return NULL; + if (unicode_ensure_utf8(unicode) == -1) { + if (psize) { + *psize = -1; } + return NULL; } if (psize) { @@ -5854,6 +5868,7 @@ unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, static int unicode_fill_utf8(PyObject *unicode) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(unicode); /* the string cannot be ASCII, or PyUnicode_UTF8() would be set */ assert(!PyUnicode_IS_ASCII(unicode)); @@ -5895,10 +5910,10 @@ unicode_fill_utf8(PyObject *unicode) PyErr_NoMemory(); return -1; } - PyUnicode_SET_UTF8(unicode, cache); - PyUnicode_SET_UTF8_LENGTH(unicode, len); memcpy(cache, start, len); cache[len] = '\0'; + PyUnicode_SET_UTF8_LENGTH(unicode, len); + PyUnicode_SET_UTF8(unicode, cache); _PyBytesWriter_Dealloc(&writer); return 0; } From 19c5134d57764d3db7b1cacec4f090c74849a5c1 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 19 Dec 2024 18:15:36 +0530 Subject: [PATCH 242/427] gh-122706: fix docs for asyncio ssl sockets (#128092) --- Doc/library/ssl.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index b7fb1fc07d199f..f07d151a885692 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2508,8 +2508,8 @@ thus several things you need to be aware of: .. seealso:: The :mod:`asyncio` module supports :ref:`non-blocking SSL sockets - ` and provides a - higher level API. It polls for events using the :mod:`selectors` module and + ` and provides a higher level :ref:`Streams API `. + It polls for events using the :mod:`selectors` module and handles :exc:`SSLWantWriteError`, :exc:`SSLWantReadError` and :exc:`BlockingIOError` exceptions. It runs the SSL handshake asynchronously as well. From ea578fc6d310c85538aefbb900a326c5c3424dd5 Mon Sep 17 00:00:00 2001 From: "RUANG (James Roy)" Date: Thu, 19 Dec 2024 21:51:21 +0800 Subject: [PATCH 243/427] gh-127688: Add `SCHED_DEADLINE` and `SCHED_NORMAL` constants to `os` module (GH-127689) --- Doc/library/os.rst | 12 ++++++++++++ Doc/whatsnew/3.14.rst | 4 ++++ .../2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst | 2 ++ Modules/posixmodule.c | 10 ++++++++++ configure | 6 ++++++ configure.ac | 2 +- pyconfig.h.in | 3 +++ 7 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index dfe5ef0726ff7d..69e6192038ab2b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5420,10 +5420,22 @@ operating system. Scheduling policy for CPU-intensive processes that tries to preserve interactivity on the rest of the computer. +.. data:: SCHED_DEADLINE + + Scheduling policy for tasks with deadline constraints. + + .. versionadded:: next + .. data:: SCHED_IDLE Scheduling policy for extremely low priority background tasks. +.. data:: SCHED_NORMAL + + Alias for :data:`SCHED_OTHER`. + + .. versionadded:: next + .. data:: SCHED_SPORADIC Scheduling policy for sporadic server programs. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 342456cbc397f3..2e43dce5e061b4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -525,6 +525,10 @@ os same process. (Contributed by Victor Stinner in :gh:`120057`.) +* Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants + to the :mod:`os` module. + (Contributed by James Roy in :gh:`127688`.) + pathlib ------- diff --git a/Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst b/Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst new file mode 100644 index 00000000000000..a22b136da72faf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-06-21-03-11.gh-issue-127688.NJqtc-.rst @@ -0,0 +1,2 @@ +Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants +to the :mod:`os` module. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2045c6065b8e7a..151d469983fafb 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -311,6 +311,10 @@ corresponding Unix manual entries for more information on calls."); # include #endif +#ifdef HAVE_LINUX_SCHED_H +# include +#endif + #if !defined(CPU_ALLOC) && defined(HAVE_SCHED_SETAFFINITY) # undef HAVE_SCHED_SETAFFINITY #endif @@ -17523,9 +17527,15 @@ all_ins(PyObject *m) #ifdef SCHED_OTHER if (PyModule_AddIntMacro(m, SCHED_OTHER)) return -1; #endif +#ifdef SCHED_DEADLINE + if (PyModule_AddIntMacro(m, SCHED_DEADLINE)) return -1; +#endif #ifdef SCHED_FIFO if (PyModule_AddIntMacro(m, SCHED_FIFO)) return -1; #endif +#ifdef SCHED_NORMAL + if (PyModule_AddIntMacro(m, SCHED_NORMAL)) return -1; +#endif #ifdef SCHED_RR if (PyModule_AddIntMacro(m, SCHED_RR)) return -1; #endif diff --git a/configure b/configure index 6df1116fc600f2..e59c7046305d46 100755 --- a/configure +++ b/configure @@ -10984,6 +10984,12 @@ if test "x$ac_cv_header_linux_soundcard_h" = xyes then : printf "%s\n" "#define HAVE_LINUX_SOUNDCARD_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "linux/sched.h" "ac_cv_header_linux_sched_h" "$ac_includes_default" +if test "x$ac_cv_header_linux_sched_h" = xyes +then : + printf "%s\n" "#define HAVE_LINUX_SCHED_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "linux/tipc.h" "ac_cv_header_linux_tipc_h" "$ac_includes_default" if test "x$ac_cv_header_linux_tipc_h" = xyes diff --git a/configure.ac b/configure.ac index 8295b59b8e45fb..074e2ce3dd3024 100644 --- a/configure.ac +++ b/configure.ac @@ -2931,7 +2931,7 @@ AC_DEFINE([STDC_HEADERS], [1], AC_CHECK_HEADERS([ \ alloca.h asm/types.h bluetooth.h conio.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ - linux/netfilter_ipv4.h linux/random.h linux/soundcard.h \ + linux/netfilter_ipv4.h linux/random.h linux/soundcard.h linux/sched.h \ linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ diff --git a/pyconfig.h.in b/pyconfig.h.in index 166c195a8c66fc..1ca83fd2f2ca1b 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -744,6 +744,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_RANDOM_H +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_SCHED_H + /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_SOUNDCARD_H From b5d1e4552f0ba40d8380368e1b099261686a89cf Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 19 Dec 2024 09:00:30 -0500 Subject: [PATCH 244/427] gh-128083: Fix macro redefinition warning in clinic. (GH-127950) --- Lib/test/clinic.test.c | 69 ++++++++++++++++++++++++++++ Modules/_io/clinic/textio.c.h | 11 ++++- Modules/clinic/_ssl.c.h | 50 +++++++++++++++++++- Tools/clinic/libclinic/parse_args.py | 3 ++ 4 files changed, 131 insertions(+), 2 deletions(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index b6ae04ecf2f8ed..e4f146c0841188 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5358,6 +5358,75 @@ static int Test_property_set_impl(TestObj *self, PyObject *value) /*[clinic end generated code: output=e4342fe9bb1d7817 input=3bc3f46a23c83a88]*/ +/*[clinic input] +@setter +Test.setter_first_with_docstr +[clinic start generated code]*/ + +#if !defined(Test_setter_first_with_docstr_DOCSTR) +# define Test_setter_first_with_docstr_DOCSTR NULL +#endif +#if defined(TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF) +# undef TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", (getter)Test_setter_first_with_docstr_get, (setter)Test_setter_first_with_docstr_set, Test_setter_first_with_docstr_DOCSTR}, +#else +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", NULL, (setter)Test_setter_first_with_docstr_set, NULL}, +#endif + +static int +Test_setter_first_with_docstr_set_impl(TestObj *self, PyObject *value); + +static int +Test_setter_first_with_docstr_set(TestObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + return_value = Test_setter_first_with_docstr_set_impl(self, value); + + return return_value; +} + +static int +Test_setter_first_with_docstr_set_impl(TestObj *self, PyObject *value) +/*[clinic end generated code: output=e4d76b558a4061db input=31a045ce11bbe961]*/ + +/*[clinic input] +@getter +Test.setter_first_with_docstr + +my silly docstring +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_setter_first_with_docstr__doc__, +"my silly docstring"); +#if defined(Test_setter_first_with_docstr_DOCSTR) +# undef Test_setter_first_with_docstr_DOCSTR +#endif +#define Test_setter_first_with_docstr_DOCSTR Test_setter_first_with_docstr__doc__ + +#if !defined(Test_setter_first_with_docstr_DOCSTR) +# define Test_setter_first_with_docstr_DOCSTR NULL +#endif +#if defined(TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF) +# undef TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", (getter)Test_setter_first_with_docstr_get, (setter)Test_setter_first_with_docstr_set, Test_setter_first_with_docstr_DOCSTR}, +#else +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", (getter)Test_setter_first_with_docstr_get, NULL, Test_setter_first_with_docstr_DOCSTR}, +#endif + +static PyObject * +Test_setter_first_with_docstr_get_impl(TestObj *self); + +static PyObject * +Test_setter_first_with_docstr_get(TestObj *self, void *Py_UNUSED(context)) +{ + return Test_setter_first_with_docstr_get_impl(self); +} + +static PyObject * +Test_setter_first_with_docstr_get_impl(TestObj *self) +/*[clinic end generated code: output=749a30266f9fb443 input=10af4e43b3cb34dc]*/ + /*[clinic input] output push output preset buffer diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 160f80ada43660..0acc1f060c811b 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -208,6 +208,9 @@ PyDoc_STRVAR(_io__TextIOBase_encoding__doc__, "Encoding of the text stream.\n" "\n" "Subclasses should override."); +#if defined(_io__TextIOBase_encoding_DOCSTR) +# undef _io__TextIOBase_encoding_DOCSTR +#endif #define _io__TextIOBase_encoding_DOCSTR _io__TextIOBase_encoding__doc__ #if !defined(_io__TextIOBase_encoding_DOCSTR) @@ -235,6 +238,9 @@ PyDoc_STRVAR(_io__TextIOBase_newlines__doc__, "Only line endings translated during reading are considered.\n" "\n" "Subclasses should override."); +#if defined(_io__TextIOBase_newlines_DOCSTR) +# undef _io__TextIOBase_newlines_DOCSTR +#endif #define _io__TextIOBase_newlines_DOCSTR _io__TextIOBase_newlines__doc__ #if !defined(_io__TextIOBase_newlines_DOCSTR) @@ -260,6 +266,9 @@ PyDoc_STRVAR(_io__TextIOBase_errors__doc__, "The error setting of the decoder or encoder.\n" "\n" "Subclasses should override."); +#if defined(_io__TextIOBase_errors_DOCSTR) +# undef _io__TextIOBase_errors_DOCSTR +#endif #define _io__TextIOBase_errors_DOCSTR _io__TextIOBase_errors__doc__ #if !defined(_io__TextIOBase_errors_DOCSTR) @@ -1281,4 +1290,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=1172c500a022c65d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=423a320f087792b9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 1ff85e32ffe5a0..becdb9cc1831fa 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -264,6 +264,9 @@ PyDoc_STRVAR(_ssl__SSLSocket_context__doc__, "This is typically used from within a callback function set by the sni_callback\n" "on the SSLContext to change the certificate information associated with the\n" "SSLSocket before the cryptographic exchange handshake messages."); +#if defined(_ssl__SSLSocket_context_DOCSTR) +# undef _ssl__SSLSocket_context_DOCSTR +#endif #define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ #if !defined(_ssl__SSLSocket_context_DOCSTR) @@ -318,6 +321,9 @@ _ssl__SSLSocket_context_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED( PyDoc_STRVAR(_ssl__SSLSocket_server_side__doc__, "Whether this is a server-side socket."); +#if defined(_ssl__SSLSocket_server_side_DOCSTR) +# undef _ssl__SSLSocket_server_side_DOCSTR +#endif #define _ssl__SSLSocket_server_side_DOCSTR _ssl__SSLSocket_server_side__doc__ #if !defined(_ssl__SSLSocket_server_side_DOCSTR) @@ -347,6 +353,9 @@ _ssl__SSLSocket_server_side_get(PySSLSocket *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl__SSLSocket_server_hostname__doc__, "The currently set server hostname (for SNI)."); +#if defined(_ssl__SSLSocket_server_hostname_DOCSTR) +# undef _ssl__SSLSocket_server_hostname_DOCSTR +#endif #define _ssl__SSLSocket_server_hostname_DOCSTR _ssl__SSLSocket_server_hostname__doc__ #if !defined(_ssl__SSLSocket_server_hostname_DOCSTR) @@ -378,6 +387,9 @@ PyDoc_STRVAR(_ssl__SSLSocket_owner__doc__, "The Python-level owner of this object.\n" "\n" "Passed as \"self\" in servername callback."); +#if defined(_ssl__SSLSocket_owner_DOCSTR) +# undef _ssl__SSLSocket_owner_DOCSTR +#endif #define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ #if !defined(_ssl__SSLSocket_owner_DOCSTR) @@ -668,6 +680,9 @@ _ssl__SSLSocket_verify_client_post_handshake(PySSLSocket *self, PyObject *Py_UNU PyDoc_STRVAR(_ssl__SSLSocket_session__doc__, "The underlying SSLSession object."); +#if defined(_ssl__SSLSocket_session_DOCSTR) +# undef _ssl__SSLSocket_session_DOCSTR +#endif #define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ #if !defined(_ssl__SSLSocket_session_DOCSTR) @@ -722,6 +737,9 @@ _ssl__SSLSocket_session_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED( PyDoc_STRVAR(_ssl__SSLSocket_session_reused__doc__, "Was the client session reused during handshake?"); +#if defined(_ssl__SSLSocket_session_reused_DOCSTR) +# undef _ssl__SSLSocket_session_reused_DOCSTR +#endif #define _ssl__SSLSocket_session_reused_DOCSTR _ssl__SSLSocket_session_reused__doc__ #if !defined(_ssl__SSLSocket_session_reused_DOCSTR) @@ -1077,6 +1095,9 @@ _ssl__SSLContext_maximum_version_set(PySSLContext *self, PyObject *value, void * PyDoc_STRVAR(_ssl__SSLContext_num_tickets__doc__, "Control the number of TLSv1.3 session tickets."); +#if defined(_ssl__SSLContext_num_tickets_DOCSTR) +# undef _ssl__SSLContext_num_tickets_DOCSTR +#endif #define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ #if !defined(_ssl__SSLContext_num_tickets_DOCSTR) @@ -1131,6 +1152,9 @@ _ssl__SSLContext_num_tickets_set(PySSLContext *self, PyObject *value, void *Py_U PyDoc_STRVAR(_ssl__SSLContext_security_level__doc__, "The current security level."); +#if defined(_ssl__SSLContext_security_level_DOCSTR) +# undef _ssl__SSLContext_security_level_DOCSTR +#endif #define _ssl__SSLContext_security_level_DOCSTR _ssl__SSLContext_security_level__doc__ #if !defined(_ssl__SSLContext_security_level_DOCSTR) @@ -1778,6 +1802,9 @@ PyDoc_STRVAR(_ssl__SSLContext_sni_callback__doc__, "with the SSLSocket, the server name as a string, and the SSLContext object.\n" "\n" "See RFC 6066 for details of the SNI extension."); +#if defined(_ssl__SSLContext_sni_callback_DOCSTR) +# undef _ssl__SSLContext_sni_callback_DOCSTR +#endif #define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ #if !defined(_ssl__SSLContext_sni_callback_DOCSTR) @@ -2100,6 +2127,9 @@ _ssl_MemoryBIO(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyDoc_STRVAR(_ssl_MemoryBIO_pending__doc__, "The number of bytes pending in the memory BIO."); +#if defined(_ssl_MemoryBIO_pending_DOCSTR) +# undef _ssl_MemoryBIO_pending_DOCSTR +#endif #define _ssl_MemoryBIO_pending_DOCSTR _ssl_MemoryBIO_pending__doc__ #if !defined(_ssl_MemoryBIO_pending_DOCSTR) @@ -2129,6 +2159,9 @@ _ssl_MemoryBIO_pending_get(PySSLMemoryBIO *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl_MemoryBIO_eof__doc__, "Whether the memory BIO is at EOF."); +#if defined(_ssl_MemoryBIO_eof_DOCSTR) +# undef _ssl_MemoryBIO_eof_DOCSTR +#endif #define _ssl_MemoryBIO_eof_DOCSTR _ssl_MemoryBIO_eof__doc__ #if !defined(_ssl_MemoryBIO_eof_DOCSTR) @@ -2262,6 +2295,9 @@ _ssl_MemoryBIO_write_eof(PySSLMemoryBIO *self, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(_ssl_SSLSession_time__doc__, "Session creation time (seconds since epoch)."); +#if defined(_ssl_SSLSession_time_DOCSTR) +# undef _ssl_SSLSession_time_DOCSTR +#endif #define _ssl_SSLSession_time_DOCSTR _ssl_SSLSession_time__doc__ #if !defined(_ssl_SSLSession_time_DOCSTR) @@ -2291,6 +2327,9 @@ _ssl_SSLSession_time_get(PySSLSession *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl_SSLSession_timeout__doc__, "Session timeout (delta in seconds)."); +#if defined(_ssl_SSLSession_timeout_DOCSTR) +# undef _ssl_SSLSession_timeout_DOCSTR +#endif #define _ssl_SSLSession_timeout_DOCSTR _ssl_SSLSession_timeout__doc__ #if !defined(_ssl_SSLSession_timeout_DOCSTR) @@ -2320,6 +2359,9 @@ _ssl_SSLSession_timeout_get(PySSLSession *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl_SSLSession_ticket_lifetime_hint__doc__, "Ticket life time hint."); +#if defined(_ssl_SSLSession_ticket_lifetime_hint_DOCSTR) +# undef _ssl_SSLSession_ticket_lifetime_hint_DOCSTR +#endif #define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR _ssl_SSLSession_ticket_lifetime_hint__doc__ #if !defined(_ssl_SSLSession_ticket_lifetime_hint_DOCSTR) @@ -2349,6 +2391,9 @@ _ssl_SSLSession_ticket_lifetime_hint_get(PySSLSession *self, void *Py_UNUSED(con PyDoc_STRVAR(_ssl_SSLSession_id__doc__, "Session ID."); +#if defined(_ssl_SSLSession_id_DOCSTR) +# undef _ssl_SSLSession_id_DOCSTR +#endif #define _ssl_SSLSession_id_DOCSTR _ssl_SSLSession_id__doc__ #if !defined(_ssl_SSLSession_id_DOCSTR) @@ -2378,6 +2423,9 @@ _ssl_SSLSession_id_get(PySSLSession *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl_SSLSession_has_ticket__doc__, "Does the session contain a ticket?"); +#if defined(_ssl_SSLSession_has_ticket_DOCSTR) +# undef _ssl_SSLSession_has_ticket_DOCSTR +#endif #define _ssl_SSLSession_has_ticket_DOCSTR _ssl_SSLSession_has_ticket__doc__ #if !defined(_ssl_SSLSession_has_ticket_DOCSTR) @@ -2830,4 +2878,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=654d6d7af659f6cd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e71f1ef621aead08 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index a57d729bec5733..ff4731e99b98d4 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -146,6 +146,9 @@ def declare_parser( GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" PyDoc_STRVAR({getset_basename}__doc__, {docstring}); + #if defined({getset_basename}_DOCSTR) + # undef {getset_basename}_DOCSTR + #endif #define {getset_basename}_DOCSTR {getset_basename}__doc__ """) IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" From b9b3e4a076caddf7876d1d4d762a117a26faffcf Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 19 Dec 2024 15:45:34 +0100 Subject: [PATCH 245/427] gh-127951: Add build option to enable pystats on Windows (GH-127952) --- .../next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst | 1 + PCbuild/build.bat | 3 +++ PCbuild/pyproject.props | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst diff --git a/Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst b/Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst new file mode 100644 index 00000000000000..0c1df0e6bd7baa --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-16-16-16-35.gh-issue-127951.lpE13-.rst @@ -0,0 +1 @@ +Add option ``--pystats`` to the Windows build to enable performance statistics collection. diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 6d3ce81651ade5..d3e3894c203a65 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -39,6 +39,7 @@ echo. --regen Regenerate all opcodes, grammar and tokens. echo. --experimental-jit Enable the experimental just-in-time compiler. echo. --experimental-jit-off Ditto but off by default (PYTHON_JIT=1 enables). echo. --experimental-jit-interpreter Enable the experimental Tier 2 interpreter. +echo. --pystats Enable PyStats collection. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -93,6 +94,7 @@ if "%~1"=="--experimental-jit" (set UseJIT=true) & (set UseTIER2=1) & shift & go if "%~1"=="--experimental-jit-off" (set UseJIT=true) & (set UseTIER2=3) & shift & goto CheckOpts if "%~1"=="--experimental-jit-interpreter" (set UseTIER2=4) & shift & goto CheckOpts if "%~1"=="--experimental-jit-interpreter-off" (set UseTIER2=6) & shift & goto CheckOpts +if "%~1"=="--pystats" (set PyStats=1) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -186,6 +188,7 @@ echo on /p:UseTestMarker=%UseTestMarker% %GITProperty%^ /p:UseJIT=%UseJIT%^ /p:UseTIER2=%UseTIER2%^ + /p:PyStats=%PyStats%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index c65341179376ea..17abfa85201a90 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -42,6 +42,8 @@ <_DebugPreprocessorDefinition>NDEBUG; <_DebugPreprocessorDefinition Condition="$(Configuration) == 'Debug'">_DEBUG; + <_PyStatsPreprocessorDefinition>PyStats; + <_PyStatsPreprocessorDefinition Condition="$(PySTATS) != ''">Py_STATS; <_PlatformPreprocessorDefinition>_WIN32; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition) @@ -50,7 +52,7 @@ $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories) - WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) + WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) _Py_USING_PGO=1;%(PreprocessorDefinitions) MaxSpeed From 7b811d0562a0bf7433165785f1549ac199610f8b Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 19 Dec 2024 10:17:15 -0500 Subject: [PATCH 246/427] gh-128008: Add `PyWeakref_IsDead()` (GH-128009) The `PyWeakref_IsDead()` function tests if a weak reference is dead without any side effects. Although you can also detect if a weak reference is dead using `PyWeakref_GetRef()`, that function returns a strong reference that must be `Py_DECREF()`'d, which can introduce side effects if the last reference is concurrently dropped (at least in the free threading build). --- Doc/c-api/weakref.rst | 9 +++++++++ Include/cpython/weakrefobject.h | 3 +++ .../2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst | 2 ++ Modules/_testcapimodule.c | 14 ++++++++++++++ Objects/weakrefobject.c | 13 +++++++++++++ 5 files changed, 41 insertions(+) create mode 100644 Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 8f233e16fb17cf..c3c6cf413dcef5 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -88,6 +88,15 @@ as much as it can. Use :c:func:`PyWeakref_GetRef` instead. +.. c:function:: int PyWeakref_IsDead(PyObject *ref) + + Test if the weak reference *ref* is dead. Returns 1 if the reference is + dead, 0 if it is alive, and -1 with an error set if *ref* is not a weak + reference object. + + .. versionadded:: 3.14 + + .. c:function:: void PyObject_ClearWeakRefs(PyObject *object) This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index 9aa1a92c413fe9..da8e77cddaca63 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -45,6 +45,9 @@ PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); #define _PyWeakref_CAST(op) \ (assert(PyWeakref_Check(op)), _Py_CAST(PyWeakReference*, (op))) +// Test if a weak reference is dead. +PyAPI_FUNC(int) PyWeakref_IsDead(PyObject *ref); + Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { PyWeakReference *ref = _PyWeakref_CAST(ref_obj); diff --git a/Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst b/Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst new file mode 100644 index 00000000000000..2349eccac8fedc --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-16-21-59-06.gh-issue-128008.fa9Jt0.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyWeakref_IsDead` function, which tests if a weak reference is +dead. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 8d86b535effb9a..f737250ac29d57 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3144,6 +3144,7 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) PyObject *ref = UNINITIALIZED_PTR; assert(PyWeakref_GetRef(weakref, &ref) == 1); assert(ref == obj); + assert(!PyWeakref_IsDead(weakref)); assert(Py_REFCNT(obj) == (refcnt + 1)); Py_DECREF(ref); @@ -3159,6 +3160,8 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(Py_REFCNT(obj) == 1); Py_DECREF(obj); + assert(PyWeakref_IsDead(weakref)); + // test PyWeakref_GET_OBJECT(), reference is dead assert(PyWeakref_GET_OBJECT(weakref) == Py_None); @@ -3181,6 +3184,12 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) PyErr_Clear(); assert(ref == NULL); + // test PyWeakRef_IsDead(), invalid type + assert(!PyErr_Occurred()); + assert(PyWeakref_IsDead(invalid_weakref) == -1); + assert(PyErr_ExceptionMatches(PyExc_TypeError)); + PyErr_Clear(); + // test PyWeakref_GetObject(), invalid type assert(PyWeakref_GetObject(invalid_weakref) == NULL); assert(PyErr_ExceptionMatches(PyExc_SystemError)); @@ -3193,6 +3202,11 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(ref == NULL); PyErr_Clear(); + // test PyWeakref_IsDead(NULL) + assert(PyWeakref_IsDead(NULL) == -1); + assert(PyErr_ExceptionMatches(PyExc_SystemError)); + PyErr_Clear(); + // test PyWeakref_GetObject(NULL) assert(PyWeakref_GetObject(NULL) == NULL); assert(PyErr_ExceptionMatches(PyExc_SystemError)); diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 9e3da1c3394d5b..0ee64ed70a63cd 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -932,6 +932,19 @@ PyWeakref_NewProxy(PyObject *ob, PyObject *callback) return (PyObject *)get_or_create_weakref(type, ob, callback); } +int +PyWeakref_IsDead(PyObject *ref) +{ + if (ref == NULL) { + PyErr_BadInternalCall(); + return -1; + } + if (!PyWeakref_Check(ref)) { + PyErr_Format(PyExc_TypeError, "expected a weakref, got %T", ref); + return -1; + } + return _PyWeakref_IS_DEAD(ref); +} int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) From d2f1d917e8b3d2dd8f35495c7632a32688883332 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 19 Dec 2024 16:59:51 +0000 Subject: [PATCH 247/427] GH-122548: Implement branch taken and not taken events for sys.monitoring (GH-122564) --- Doc/c-api/monitoring.rst | 12 +- Doc/library/sys.monitoring.rst | 29 +- Doc/whatsnew/3.13.rst | 2 +- Doc/whatsnew/3.14.rst | 14 + Include/cpython/code.h | 6 +- Include/cpython/monitoring.h | 43 +- Include/internal/pycore_code.h | 2 + Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 25 +- Include/internal/pycore_opcode_utils.h | 6 + Include/internal/pycore_uop_ids.h | 1 + Include/opcode_ids.h | 198 ++++----- Lib/_opcode_metadata.py | 198 ++++----- Lib/test/test_compiler_codegen.py | 2 + Lib/test/test_dis.py | 391 +++++++++--------- Lib/test/test_monitoring.py | 67 ++- ...-12-13-14-21-04.gh-issue-122548.hq3Vud.rst | 4 + Modules/_testcapi/monitoring.c | 25 +- Objects/codeobject.c | 7 + Programs/test_frozenmain.h | 66 +-- Python/bytecodes.c | 60 ++- Python/codegen.c | 9 +- Python/executor_cases.c.h | 2 + Python/flowgraph.c | 6 + Python/generated_cases.c.h | 74 ++-- Python/instrumentation.c | 321 +++++++++++--- Python/opcode_targets.h | 4 +- Python/optimizer_cases.c.h | 2 + Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 + 29 files changed, 998 insertions(+), 583 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 51d866cfd47469..bda6cd271197d0 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -75,9 +75,14 @@ See :mod:`sys.monitoring` for descriptions of the events. Fire a ``JUMP`` event. -.. c:function:: int PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) +.. c:function:: int PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) - Fire a ``BRANCH`` event. + Fire a ``BRANCH_LEFT`` event. + + +.. c:function:: int PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH_RIGHT`` event. .. c:function:: int PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval) @@ -168,7 +173,8 @@ would typically correspond to a python function. ================================================== ===================================== Macro Event ================================================== ===================================== - .. c:macro:: PY_MONITORING_EVENT_BRANCH :monitoring-event:`BRANCH` + .. c:macro:: PY_MONITORING_EVENT_BRANCH_LEFT :monitoring-event:`BRANCH_LEFT` + .. c:macro:: PY_MONITORING_EVENT_BRANCH_RIGHT :monitoring-event:`BRANCH_RIGHT` .. c:macro:: PY_MONITORING_EVENT_CALL :monitoring-event:`CALL` .. c:macro:: PY_MONITORING_EVENT_C_RAISE :monitoring-event:`C_RAISE` .. c:macro:: PY_MONITORING_EVENT_C_RETURN :monitoring-event:`C_RETURN` diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index f7140af2494898..cfdcdf2e2df476 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -79,9 +79,17 @@ Events The following events are supported: -.. monitoring-event:: BRANCH +.. monitoring-event:: BRANCH_LEFT - A conditional branch is taken (or not). + A conditional branch goes left. + + It is up to the tool to determine how to present "left" and "right" branches. + There is no guarantee which branch is "left" and which is "right", except + that it will be consistent for the duration of the program. + +.. monitoring-event:: BRANCH_RIGHT + + A conditional branch goes right. .. monitoring-event:: CALL @@ -180,9 +188,20 @@ The local events are: * :monitoring-event:`LINE` * :monitoring-event:`INSTRUCTION` * :monitoring-event:`JUMP` -* :monitoring-event:`BRANCH` +* :monitoring-event:`BRANCH_LEFT` +* :monitoring-event:`BRANCH_RIGHT` * :monitoring-event:`STOP_ITERATION` +Deprecated event +'''''''''''''''' + +* ``BRANCH`` + +The ``BRANCH`` event is deprecated in 3.14. +Using :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` +events will give much better performance as they can be disabled +independently. + Ancillary events '''''''''''''''' @@ -357,13 +376,11 @@ Different events will provide the callback function with different arguments, as func(code: CodeType, line_number: int) -> DISABLE | Any -* :monitoring-event:`BRANCH` and :monitoring-event:`JUMP`:: +* :monitoring-event:`BRANCH_LEFT`, :monitoring-event:`BRANCH_RIGHT` and :monitoring-event:`JUMP`:: func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any Note that the *destination_offset* is where the code will next execute. - For an untaken branch this will be the offset of the instruction following - the branch. * :monitoring-event:`INSTRUCTION`:: diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 9f6d98b9950d19..45cc1b5bad9b18 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1971,7 +1971,7 @@ New Features * :c:func:`PyMonitoring_FireCallEvent` * :c:func:`PyMonitoring_FireLineEvent` * :c:func:`PyMonitoring_FireJumpEvent` - * :c:func:`PyMonitoring_FireBranchEvent` + * ``PyMonitoring_FireBranchEvent`` * :c:func:`PyMonitoring_FireCReturnEvent` * :c:func:`PyMonitoring_FirePyThrowEvent` * :c:func:`PyMonitoring_FireRaiseEvent` diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2e43dce5e061b4..97a37a82f76b9b 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -603,6 +603,11 @@ sys which only exists in specialized builds of Python, may now return objects from other interpreters than the one it's called in. +sys.monitoring +-------------- + +Two new events are added: :monitoring-event:`BRANCH_LEFT` and +:monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. tkinter ------- @@ -1144,6 +1149,11 @@ New features a :exc:`UnicodeError` object. (Contributed by Bénédikt Tran in :gh:`127691`.) +* Add :c:func:`PyMonitoring_FireBranchLeftEvent` and + :c:func:`PyMonitoring_FireBranchRightEvent` for generating + :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` + events, respectively. + Porting to Python 3.14 ---------------------- @@ -1177,6 +1187,10 @@ Deprecated .. include:: ../deprecations/c-api-pending-removal-in-future.rst +* The ``PyMonitoring_FireBranchEvent`` function is deprecated and should + be replaced with calls to :c:func:`PyMonitoring_FireBranchLeftEvent` + and :c:func:`PyMonitoring_FireBranchRightEvent`. + Removed ------- diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 3899d4269233a1..c3c0165d556ead 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -11,11 +11,11 @@ extern "C" { /* Total tool ids available */ #define _PY_MONITORING_TOOL_IDS 8 /* Count of all local monitoring events */ -#define _PY_MONITORING_LOCAL_EVENTS 10 +#define _PY_MONITORING_LOCAL_EVENTS 11 /* Count of all "real" monitoring events (not derived from other events) */ -#define _PY_MONITORING_UNGROUPED_EVENTS 15 +#define _PY_MONITORING_UNGROUPED_EVENTS 16 /* Count of all monitoring events */ -#define _PY_MONITORING_EVENTS 17 +#define _PY_MONITORING_EVENTS 19 /* Tables of which tools are active for each monitored event. */ typedef struct _Py_LocalMonitors { diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index 797ba51246b1c6..ce92942404c9f7 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -13,25 +13,27 @@ #define PY_MONITORING_EVENT_LINE 5 #define PY_MONITORING_EVENT_INSTRUCTION 6 #define PY_MONITORING_EVENT_JUMP 7 -#define PY_MONITORING_EVENT_BRANCH 8 -#define PY_MONITORING_EVENT_STOP_ITERATION 9 +#define PY_MONITORING_EVENT_BRANCH_LEFT 8 +#define PY_MONITORING_EVENT_BRANCH_RIGHT 9 +#define PY_MONITORING_EVENT_STOP_ITERATION 10 #define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ ((ev) < _PY_MONITORING_LOCAL_EVENTS) /* Other events, mainly exceptions */ -#define PY_MONITORING_EVENT_RAISE 10 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 -#define PY_MONITORING_EVENT_PY_UNWIND 12 -#define PY_MONITORING_EVENT_PY_THROW 13 -#define PY_MONITORING_EVENT_RERAISE 14 +#define PY_MONITORING_EVENT_RAISE 11 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 +#define PY_MONITORING_EVENT_PY_UNWIND 13 +#define PY_MONITORING_EVENT_PY_THROW 14 +#define PY_MONITORING_EVENT_RERAISE 15 /* Ancillary events */ -#define PY_MONITORING_EVENT_C_RETURN 15 -#define PY_MONITORING_EVENT_C_RAISE 16 +#define PY_MONITORING_EVENT_C_RETURN 16 +#define PY_MONITORING_EVENT_C_RAISE 17 +#define PY_MONITORING_EVENT_BRANCH 18 typedef struct _PyMonitoringState { @@ -74,10 +76,18 @@ PyAPI_FUNC(int) _PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset); -PyAPI_FUNC(int) +Py_DEPRECATED(3.14) PyAPI_FUNC(int) _PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset); +PyAPI_FUNC(int) +_PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + PyAPI_FUNC(int) _PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval); @@ -174,12 +184,21 @@ PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t } static inline int -PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, +PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireBranchRightEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) { _PYMONITORING_IF_ACTIVE( state, - _PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset)); + _PyMonitoring_FireBranchLeftEvent(state, codelike, offset, target_offset)); } static inline int diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index d607a54aa4a2f5..d97fe81a2fc54a 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -603,6 +603,8 @@ extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); +extern PyObject *_PyInstrumentation_BranchesIterator(PyCodeObject *code); + struct _PyCode8 _PyCode_DEF(8); PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 14e29576875c6e..079a9befcd4c5e 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -262,6 +262,7 @@ Known values: Python 3.14a1 3607 (Add pseudo instructions JUMP_IF_TRUE/FALSE) Python 3.14a1 3608 (Add support for slices) Python 3.14a2 3609 (Add LOAD_SMALL_INT and LOAD_CONST_IMMORTAL instructions, remove RETURN_CONST) + Python 3.14a3 3610 (Add NOT_TAKEN instruction) Python 3.15 will start with 3650 @@ -274,7 +275,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3609 +#define PYC_MAGIC_NUMBER 3611 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d2ae8928a8fe8f..e09fff062b5202 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -243,6 +243,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case INSTRUMENTED_LOAD_SUPER_ATTR: return 0; + case INSTRUMENTED_NOT_TAKEN: + return 0; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -367,6 +369,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case NOP: return 0; + case NOT_TAKEN: + return 0; case POP_BLOCK: return 0; case POP_EXCEPT: @@ -702,6 +706,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case INSTRUMENTED_LOAD_SUPER_ATTR: return 0; + case INSTRUMENTED_NOT_TAKEN: + return 0; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -826,6 +832,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2; case NOP: return 0; + case NOT_TAKEN: + return 0; case POP_BLOCK: return 0; case POP_EXCEPT: @@ -1387,6 +1395,10 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { *effect = 0; return 0; } + case INSTRUMENTED_NOT_TAKEN: { + *effect = 0; + return 0; + } case INSTRUMENTED_POP_JUMP_IF_FALSE: { *effect = 0; return 0; @@ -1635,6 +1647,10 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { *effect = 0; return 0; } + case NOT_TAKEN: { + *effect = 0; + return 0; + } case POP_BLOCK: { *effect = 0; return 0; @@ -2043,6 +2059,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IXC, 0 }, + [INSTRUMENTED_NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, @@ -2100,6 +2117,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [NOT_TAKEN] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -2304,6 +2322,7 @@ _PyOpcode_macro_expansion[256] = { [MATCH_MAPPING] = { .nuops = 1, .uops = { { _MATCH_MAPPING, 0, 0 } } }, [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { _MATCH_SEQUENCE, 0, 0 } } }, [NOP] = { .nuops = 1, .uops = { { _NOP, 0, 0 } } }, + [NOT_TAKEN] = { .nuops = 1, .uops = { { _NOP, 0, 0 } } }, [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, 0, 0 } } }, [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 9, 1 } } }, [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 9, 1 } } }, @@ -2462,6 +2481,7 @@ const char *_PyOpcode_OpName[266] = { [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", + [INSTRUMENTED_NOT_TAKEN] = "INSTRUMENTED_NOT_TAKEN", [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", @@ -2524,6 +2544,7 @@ const char *_PyOpcode_OpName[266] = { [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [NOP] = "NOP", + [NOT_TAKEN] = "NOT_TAKEN", [POP_BLOCK] = "POP_BLOCK", [POP_EXCEPT] = "POP_EXCEPT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", @@ -2718,6 +2739,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, @@ -2775,6 +2797,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [MATCH_MAPPING] = MATCH_MAPPING, [MATCH_SEQUENCE] = MATCH_SEQUENCE, [NOP] = NOP, + [NOT_TAKEN] = NOT_TAKEN, [POP_EXCEPT] = POP_EXCEPT, [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, @@ -2833,7 +2856,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 116: \ case 117: \ case 118: \ case 119: \ @@ -2874,7 +2896,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 233: \ case 234: \ case 235: \ - case 236: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index c6ce7e65a65d4b..0872231d1f2d11 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -45,6 +45,12 @@ extern "C" { (opcode) == JUMP_BACKWARD || \ (opcode) == JUMP_BACKWARD_NO_INTERRUPT) +#define IS_CONDITIONAL_JUMP_OPCODE(opcode) \ + ((opcode) == POP_JUMP_IF_FALSE || \ + (opcode) == POP_JUMP_IF_TRUE || \ + (opcode) == POP_JUMP_IF_NONE || \ + (opcode) == POP_JUMP_IF_NOT_NONE) + #define IS_SCOPE_EXIT_OPCODE(opcode) \ ((opcode) == RETURN_VALUE || \ (opcode) == RAISE_VARARGS || \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 45563585dd5681..5fc57e48f500d0 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -155,6 +155,7 @@ extern "C" { #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD #define _INSTRUMENTED_LINE INSTRUMENTED_LINE #define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _INSTRUMENTED_NOT_TAKEN INSTRUMENTED_NOT_TAKEN #define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index ce3d23eaa6d56d..3cd189b93dd9d6 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -38,94 +38,95 @@ extern "C" { #define MATCH_MAPPING 25 #define MATCH_SEQUENCE 26 #define NOP 27 -#define POP_EXCEPT 28 -#define POP_TOP 29 -#define PUSH_EXC_INFO 30 -#define PUSH_NULL 31 -#define RETURN_GENERATOR 32 -#define RETURN_VALUE 33 -#define SETUP_ANNOTATIONS 34 -#define STORE_SLICE 35 -#define STORE_SUBSCR 36 -#define TO_BOOL 37 -#define UNARY_INVERT 38 -#define UNARY_NEGATIVE 39 -#define UNARY_NOT 40 -#define WITH_EXCEPT_START 41 -#define BINARY_OP 42 -#define BUILD_LIST 43 -#define BUILD_MAP 44 -#define BUILD_SET 45 -#define BUILD_SLICE 46 -#define BUILD_STRING 47 -#define BUILD_TUPLE 48 -#define CALL 49 -#define CALL_FUNCTION_EX 50 -#define CALL_INTRINSIC_1 51 -#define CALL_INTRINSIC_2 52 -#define CALL_KW 53 -#define COMPARE_OP 54 -#define CONTAINS_OP 55 -#define CONVERT_VALUE 56 -#define COPY 57 -#define COPY_FREE_VARS 58 -#define DELETE_ATTR 59 -#define DELETE_DEREF 60 -#define DELETE_FAST 61 -#define DELETE_GLOBAL 62 -#define DELETE_NAME 63 -#define DICT_MERGE 64 -#define DICT_UPDATE 65 -#define EXTENDED_ARG 66 -#define FOR_ITER 67 -#define GET_AWAITABLE 68 -#define IMPORT_FROM 69 -#define IMPORT_NAME 70 -#define IS_OP 71 -#define JUMP_BACKWARD 72 -#define JUMP_BACKWARD_NO_INTERRUPT 73 -#define JUMP_FORWARD 74 -#define LIST_APPEND 75 -#define LIST_EXTEND 76 -#define LOAD_ATTR 77 -#define LOAD_COMMON_CONSTANT 78 -#define LOAD_CONST 79 -#define LOAD_DEREF 80 -#define LOAD_FAST 81 -#define LOAD_FAST_AND_CLEAR 82 -#define LOAD_FAST_CHECK 83 -#define LOAD_FAST_LOAD_FAST 84 -#define LOAD_FROM_DICT_OR_DEREF 85 -#define LOAD_FROM_DICT_OR_GLOBALS 86 -#define LOAD_GLOBAL 87 -#define LOAD_NAME 88 -#define LOAD_SMALL_INT 89 -#define LOAD_SPECIAL 90 -#define LOAD_SUPER_ATTR 91 -#define MAKE_CELL 92 -#define MAP_ADD 93 -#define MATCH_CLASS 94 -#define POP_JUMP_IF_FALSE 95 -#define POP_JUMP_IF_NONE 96 -#define POP_JUMP_IF_NOT_NONE 97 -#define POP_JUMP_IF_TRUE 98 -#define RAISE_VARARGS 99 -#define RERAISE 100 -#define SEND 101 -#define SET_ADD 102 -#define SET_FUNCTION_ATTRIBUTE 103 -#define SET_UPDATE 104 -#define STORE_ATTR 105 -#define STORE_DEREF 106 -#define STORE_FAST 107 -#define STORE_FAST_LOAD_FAST 108 -#define STORE_FAST_STORE_FAST 109 -#define STORE_GLOBAL 110 -#define STORE_NAME 111 -#define SWAP 112 -#define UNPACK_EX 113 -#define UNPACK_SEQUENCE 114 -#define YIELD_VALUE 115 +#define NOT_TAKEN 28 +#define POP_EXCEPT 29 +#define POP_TOP 30 +#define PUSH_EXC_INFO 31 +#define PUSH_NULL 32 +#define RETURN_GENERATOR 33 +#define RETURN_VALUE 34 +#define SETUP_ANNOTATIONS 35 +#define STORE_SLICE 36 +#define STORE_SUBSCR 37 +#define TO_BOOL 38 +#define UNARY_INVERT 39 +#define UNARY_NEGATIVE 40 +#define UNARY_NOT 41 +#define WITH_EXCEPT_START 42 +#define BINARY_OP 43 +#define BUILD_LIST 44 +#define BUILD_MAP 45 +#define BUILD_SET 46 +#define BUILD_SLICE 47 +#define BUILD_STRING 48 +#define BUILD_TUPLE 49 +#define CALL 50 +#define CALL_FUNCTION_EX 51 +#define CALL_INTRINSIC_1 52 +#define CALL_INTRINSIC_2 53 +#define CALL_KW 54 +#define COMPARE_OP 55 +#define CONTAINS_OP 56 +#define CONVERT_VALUE 57 +#define COPY 58 +#define COPY_FREE_VARS 59 +#define DELETE_ATTR 60 +#define DELETE_DEREF 61 +#define DELETE_FAST 62 +#define DELETE_GLOBAL 63 +#define DELETE_NAME 64 +#define DICT_MERGE 65 +#define DICT_UPDATE 66 +#define EXTENDED_ARG 67 +#define FOR_ITER 68 +#define GET_AWAITABLE 69 +#define IMPORT_FROM 70 +#define IMPORT_NAME 71 +#define IS_OP 72 +#define JUMP_BACKWARD 73 +#define JUMP_BACKWARD_NO_INTERRUPT 74 +#define JUMP_FORWARD 75 +#define LIST_APPEND 76 +#define LIST_EXTEND 77 +#define LOAD_ATTR 78 +#define LOAD_COMMON_CONSTANT 79 +#define LOAD_CONST 80 +#define LOAD_DEREF 81 +#define LOAD_FAST 82 +#define LOAD_FAST_AND_CLEAR 83 +#define LOAD_FAST_CHECK 84 +#define LOAD_FAST_LOAD_FAST 85 +#define LOAD_FROM_DICT_OR_DEREF 86 +#define LOAD_FROM_DICT_OR_GLOBALS 87 +#define LOAD_GLOBAL 88 +#define LOAD_NAME 89 +#define LOAD_SMALL_INT 90 +#define LOAD_SPECIAL 91 +#define LOAD_SUPER_ATTR 92 +#define MAKE_CELL 93 +#define MAP_ADD 94 +#define MATCH_CLASS 95 +#define POP_JUMP_IF_FALSE 96 +#define POP_JUMP_IF_NONE 97 +#define POP_JUMP_IF_NOT_NONE 98 +#define POP_JUMP_IF_TRUE 99 +#define RAISE_VARARGS 100 +#define RERAISE 101 +#define SEND 102 +#define SET_ADD 103 +#define SET_FUNCTION_ATTRIBUTE 104 +#define SET_UPDATE 105 +#define STORE_ATTR 106 +#define STORE_DEREF 107 +#define STORE_FAST 108 +#define STORE_FAST_LOAD_FAST 109 +#define STORE_FAST_STORE_FAST 110 +#define STORE_GLOBAL 111 +#define STORE_NAME 112 +#define SWAP 113 +#define UNPACK_EX 114 +#define UNPACK_SEQUENCE 115 +#define YIELD_VALUE 116 #define RESUME 149 #define BINARY_OP_ADD_FLOAT 150 #define BINARY_OP_ADD_INT 151 @@ -205,14 +206,15 @@ extern "C" { #define UNPACK_SEQUENCE_LIST 225 #define UNPACK_SEQUENCE_TUPLE 226 #define UNPACK_SEQUENCE_TWO_TUPLE 227 -#define INSTRUMENTED_END_FOR 237 -#define INSTRUMENTED_END_SEND 238 -#define INSTRUMENTED_LOAD_SUPER_ATTR 239 -#define INSTRUMENTED_FOR_ITER 240 -#define INSTRUMENTED_CALL_KW 241 -#define INSTRUMENTED_CALL_FUNCTION_EX 242 -#define INSTRUMENTED_INSTRUCTION 243 -#define INSTRUMENTED_JUMP_FORWARD 244 +#define INSTRUMENTED_END_FOR 236 +#define INSTRUMENTED_END_SEND 237 +#define INSTRUMENTED_LOAD_SUPER_ATTR 238 +#define INSTRUMENTED_FOR_ITER 239 +#define INSTRUMENTED_CALL_KW 240 +#define INSTRUMENTED_CALL_FUNCTION_EX 241 +#define INSTRUMENTED_INSTRUCTION 242 +#define INSTRUMENTED_JUMP_FORWARD 243 +#define INSTRUMENTED_NOT_TAKEN 244 #define INSTRUMENTED_POP_JUMP_IF_TRUE 245 #define INSTRUMENTED_POP_JUMP_IF_FALSE 246 #define INSTRUMENTED_POP_JUMP_IF_NONE 247 @@ -235,9 +237,9 @@ extern "C" { #define SETUP_WITH 264 #define STORE_FAST_MAYBE_NULL 265 -#define HAVE_ARGUMENT 41 +#define HAVE_ARGUMENT 42 #define MIN_SPECIALIZED_OPCODE 150 -#define MIN_INSTRUMENTED_OPCODE 237 +#define MIN_INSTRUMENTED_OPCODE 236 #ifdef __cplusplus } diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index cda3c340c322f3..dada2cb5fa033f 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -231,102 +231,104 @@ 'MATCH_MAPPING': 25, 'MATCH_SEQUENCE': 26, 'NOP': 27, - 'POP_EXCEPT': 28, - 'POP_TOP': 29, - 'PUSH_EXC_INFO': 30, - 'PUSH_NULL': 31, - 'RETURN_GENERATOR': 32, - 'RETURN_VALUE': 33, - 'SETUP_ANNOTATIONS': 34, - 'STORE_SLICE': 35, - 'STORE_SUBSCR': 36, - 'TO_BOOL': 37, - 'UNARY_INVERT': 38, - 'UNARY_NEGATIVE': 39, - 'UNARY_NOT': 40, - 'WITH_EXCEPT_START': 41, - 'BINARY_OP': 42, - 'BUILD_LIST': 43, - 'BUILD_MAP': 44, - 'BUILD_SET': 45, - 'BUILD_SLICE': 46, - 'BUILD_STRING': 47, - 'BUILD_TUPLE': 48, - 'CALL': 49, - 'CALL_FUNCTION_EX': 50, - 'CALL_INTRINSIC_1': 51, - 'CALL_INTRINSIC_2': 52, - 'CALL_KW': 53, - 'COMPARE_OP': 54, - 'CONTAINS_OP': 55, - 'CONVERT_VALUE': 56, - 'COPY': 57, - 'COPY_FREE_VARS': 58, - 'DELETE_ATTR': 59, - 'DELETE_DEREF': 60, - 'DELETE_FAST': 61, - 'DELETE_GLOBAL': 62, - 'DELETE_NAME': 63, - 'DICT_MERGE': 64, - 'DICT_UPDATE': 65, - 'EXTENDED_ARG': 66, - 'FOR_ITER': 67, - 'GET_AWAITABLE': 68, - 'IMPORT_FROM': 69, - 'IMPORT_NAME': 70, - 'IS_OP': 71, - 'JUMP_BACKWARD': 72, - 'JUMP_BACKWARD_NO_INTERRUPT': 73, - 'JUMP_FORWARD': 74, - 'LIST_APPEND': 75, - 'LIST_EXTEND': 76, - 'LOAD_ATTR': 77, - 'LOAD_COMMON_CONSTANT': 78, - 'LOAD_CONST': 79, - 'LOAD_DEREF': 80, - 'LOAD_FAST': 81, - 'LOAD_FAST_AND_CLEAR': 82, - 'LOAD_FAST_CHECK': 83, - 'LOAD_FAST_LOAD_FAST': 84, - 'LOAD_FROM_DICT_OR_DEREF': 85, - 'LOAD_FROM_DICT_OR_GLOBALS': 86, - 'LOAD_GLOBAL': 87, - 'LOAD_NAME': 88, - 'LOAD_SMALL_INT': 89, - 'LOAD_SPECIAL': 90, - 'LOAD_SUPER_ATTR': 91, - 'MAKE_CELL': 92, - 'MAP_ADD': 93, - 'MATCH_CLASS': 94, - 'POP_JUMP_IF_FALSE': 95, - 'POP_JUMP_IF_NONE': 96, - 'POP_JUMP_IF_NOT_NONE': 97, - 'POP_JUMP_IF_TRUE': 98, - 'RAISE_VARARGS': 99, - 'RERAISE': 100, - 'SEND': 101, - 'SET_ADD': 102, - 'SET_FUNCTION_ATTRIBUTE': 103, - 'SET_UPDATE': 104, - 'STORE_ATTR': 105, - 'STORE_DEREF': 106, - 'STORE_FAST': 107, - 'STORE_FAST_LOAD_FAST': 108, - 'STORE_FAST_STORE_FAST': 109, - 'STORE_GLOBAL': 110, - 'STORE_NAME': 111, - 'SWAP': 112, - 'UNPACK_EX': 113, - 'UNPACK_SEQUENCE': 114, - 'YIELD_VALUE': 115, - 'INSTRUMENTED_END_FOR': 237, - 'INSTRUMENTED_END_SEND': 238, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 239, - 'INSTRUMENTED_FOR_ITER': 240, - 'INSTRUMENTED_CALL_KW': 241, - 'INSTRUMENTED_CALL_FUNCTION_EX': 242, - 'INSTRUMENTED_INSTRUCTION': 243, - 'INSTRUMENTED_JUMP_FORWARD': 244, + 'NOT_TAKEN': 28, + 'POP_EXCEPT': 29, + 'POP_TOP': 30, + 'PUSH_EXC_INFO': 31, + 'PUSH_NULL': 32, + 'RETURN_GENERATOR': 33, + 'RETURN_VALUE': 34, + 'SETUP_ANNOTATIONS': 35, + 'STORE_SLICE': 36, + 'STORE_SUBSCR': 37, + 'TO_BOOL': 38, + 'UNARY_INVERT': 39, + 'UNARY_NEGATIVE': 40, + 'UNARY_NOT': 41, + 'WITH_EXCEPT_START': 42, + 'BINARY_OP': 43, + 'BUILD_LIST': 44, + 'BUILD_MAP': 45, + 'BUILD_SET': 46, + 'BUILD_SLICE': 47, + 'BUILD_STRING': 48, + 'BUILD_TUPLE': 49, + 'CALL': 50, + 'CALL_FUNCTION_EX': 51, + 'CALL_INTRINSIC_1': 52, + 'CALL_INTRINSIC_2': 53, + 'CALL_KW': 54, + 'COMPARE_OP': 55, + 'CONTAINS_OP': 56, + 'CONVERT_VALUE': 57, + 'COPY': 58, + 'COPY_FREE_VARS': 59, + 'DELETE_ATTR': 60, + 'DELETE_DEREF': 61, + 'DELETE_FAST': 62, + 'DELETE_GLOBAL': 63, + 'DELETE_NAME': 64, + 'DICT_MERGE': 65, + 'DICT_UPDATE': 66, + 'EXTENDED_ARG': 67, + 'FOR_ITER': 68, + 'GET_AWAITABLE': 69, + 'IMPORT_FROM': 70, + 'IMPORT_NAME': 71, + 'IS_OP': 72, + 'JUMP_BACKWARD': 73, + 'JUMP_BACKWARD_NO_INTERRUPT': 74, + 'JUMP_FORWARD': 75, + 'LIST_APPEND': 76, + 'LIST_EXTEND': 77, + 'LOAD_ATTR': 78, + 'LOAD_COMMON_CONSTANT': 79, + 'LOAD_CONST': 80, + 'LOAD_DEREF': 81, + 'LOAD_FAST': 82, + 'LOAD_FAST_AND_CLEAR': 83, + 'LOAD_FAST_CHECK': 84, + 'LOAD_FAST_LOAD_FAST': 85, + 'LOAD_FROM_DICT_OR_DEREF': 86, + 'LOAD_FROM_DICT_OR_GLOBALS': 87, + 'LOAD_GLOBAL': 88, + 'LOAD_NAME': 89, + 'LOAD_SMALL_INT': 90, + 'LOAD_SPECIAL': 91, + 'LOAD_SUPER_ATTR': 92, + 'MAKE_CELL': 93, + 'MAP_ADD': 94, + 'MATCH_CLASS': 95, + 'POP_JUMP_IF_FALSE': 96, + 'POP_JUMP_IF_NONE': 97, + 'POP_JUMP_IF_NOT_NONE': 98, + 'POP_JUMP_IF_TRUE': 99, + 'RAISE_VARARGS': 100, + 'RERAISE': 101, + 'SEND': 102, + 'SET_ADD': 103, + 'SET_FUNCTION_ATTRIBUTE': 104, + 'SET_UPDATE': 105, + 'STORE_ATTR': 106, + 'STORE_DEREF': 107, + 'STORE_FAST': 108, + 'STORE_FAST_LOAD_FAST': 109, + 'STORE_FAST_STORE_FAST': 110, + 'STORE_GLOBAL': 111, + 'STORE_NAME': 112, + 'SWAP': 113, + 'UNPACK_EX': 114, + 'UNPACK_SEQUENCE': 115, + 'YIELD_VALUE': 116, + 'INSTRUMENTED_END_FOR': 236, + 'INSTRUMENTED_END_SEND': 237, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 238, + 'INSTRUMENTED_FOR_ITER': 239, + 'INSTRUMENTED_CALL_KW': 240, + 'INSTRUMENTED_CALL_FUNCTION_EX': 241, + 'INSTRUMENTED_INSTRUCTION': 242, + 'INSTRUMENTED_JUMP_FORWARD': 243, + 'INSTRUMENTED_NOT_TAKEN': 244, 'INSTRUMENTED_POP_JUMP_IF_TRUE': 245, 'INSTRUMENTED_POP_JUMP_IF_FALSE': 246, 'INSTRUMENTED_POP_JUMP_IF_NONE': 247, @@ -348,5 +350,5 @@ 'STORE_FAST_MAYBE_NULL': 265, } -HAVE_ARGUMENT = 41 -MIN_INSTRUMENTED_OPCODE = 237 +HAVE_ARGUMENT = 42 +MIN_INSTRUMENTED_OPCODE = 236 diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 2dd7cf65ee3c2a..f8c4fc14c91ebe 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -29,6 +29,7 @@ def test_if_expression(self): ('LOAD_CONST', 0, 1), ('TO_BOOL', 0, 1), ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1), + ('NOT_TAKEN', None, 1), ('LOAD_SMALL_INT', 42, 1), ('JUMP_NO_INTERRUPT', exit_lbl := self.Label()), false_lbl, @@ -49,6 +50,7 @@ def test_for_loop(self): ('GET_ITER', None, 1), loop_lbl := self.Label(), ('FOR_ITER', exit_lbl := self.Label(), 1), + ('NOT_TAKEN', None, 1), ('NOP', None, 1, 1), ('STORE_NAME', 1, 1), ('LOAD_NAME', 2, 2), diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index c719f571152d61..955a3e4cb9e4f7 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -175,10 +175,11 @@ def bug708901(): %3d CALL 2 GET_ITER - L1: FOR_ITER 3 (to L2) + L1: FOR_ITER 4 (to L2) + NOT_TAKEN STORE_FAST 0 (res) -%3d JUMP_BACKWARD 5 (to L1) +%3d JUMP_BACKWARD 6 (to L1) %3d L2: END_FOR POP_TOP @@ -200,7 +201,8 @@ def bug1333982(x=[]): dis_bug1333982 = """\ %3d RESUME 0 -%3d LOAD_COMMON_CONSTANT 0 (AssertionError) +%3d NOT_TAKEN + LOAD_COMMON_CONSTANT 0 (AssertionError) LOAD_CONST 0 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION LOAD_FAST 0 (x) @@ -432,7 +434,7 @@ def foo(a: int, b: str) -> str: 1 LOAD_SMALL_INT 0 STORE_NAME 0 (x) - 2 L1: NOP + 2 L1: NOT_TAKEN 3 LOAD_NAME 0 (x) LOAD_SMALL_INT 1 @@ -458,7 +460,8 @@ def foo(a: int, b: str) -> str: %4d LOAD_GLOBAL 0 (Exception) CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 23 (to L7) + POP_JUMP_IF_FALSE 24 (to L7) + NOT_TAKEN STORE_FAST 0 (e) %4d L4: LOAD_FAST 0 (e) @@ -555,7 +558,8 @@ def _with(c): %4d L3: PUSH_EXC_INFO WITH_EXCEPT_START TO_BOOL - POP_JUMP_IF_TRUE 1 (to L4) + POP_JUMP_IF_TRUE 2 (to L4) + NOT_TAKEN RERAISE 2 L4: POP_TOP L5: POP_EXCEPT @@ -645,7 +649,8 @@ async def _asyncwith(c): L20: CLEANUP_THROW L21: END_SEND TO_BOOL - POP_JUMP_IF_TRUE 1 (to L22) + POP_JUMP_IF_TRUE 2 (to L22) + NOT_TAKEN RERAISE 2 L22: POP_TOP L23: POP_EXCEPT @@ -839,7 +844,8 @@ def foo(x): L1: RESUME 0 LOAD_FAST 0 (.0) GET_ITER - L2: FOR_ITER 10 (to L3) + L2: FOR_ITER 11 (to L3) + NOT_TAKEN STORE_FAST 1 (z) LOAD_DEREF 2 (x) LOAD_FAST 1 (z) @@ -847,7 +853,7 @@ def foo(x): YIELD_VALUE 0 RESUME 5 POP_TOP - JUMP_BACKWARD 12 (to L2) + JUMP_BACKWARD 13 (to L2) L3: END_FOR POP_TOP LOAD_CONST 0 (None) @@ -893,14 +899,15 @@ def loop_test(): LOAD_SMALL_INT 3 BINARY_OP 5 (*) GET_ITER - L1: FOR_ITER_LIST 14 (to L2) + L1: FOR_ITER_LIST 15 (to L2) + NOT_TAKEN STORE_FAST 0 (i) %3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST 0 (i) CALL_PY_GENERAL 1 POP_TOP - JUMP_BACKWARD 16 (to L1) + JUMP_BACKWARD 17 (to L1) %3d L2: END_FOR POP_TOP @@ -1699,204 +1706,214 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=92, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=92, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_TUPLE', opcode=48, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=49, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=107, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=80, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=1, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=1, argval=1, argrepr='', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_LIST', opcode=43, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_MAP', opcode=44, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=2, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=108, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=81, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=1, argval=1, argrepr='', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_LIST', opcode=44, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_MAP', opcode=45, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=2, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=58, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=92, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=92, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='COPY_FREE_VARS', opcode=59, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_TUPLE', opcode=48, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=49, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=103, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=107, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=80, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=108, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=81, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=58, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='COPY_FREE_VARS', opcode=59, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=80, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=80, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=84, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=36, start_offset=36, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=81, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=85, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=36, start_offset=36, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='GET_ITER', opcode=16, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='FOR_ITER', opcode=67, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=107, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=4, argval=4, argrepr='', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=54, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=72, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=6, argval=6, argrepr='', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=54, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=72, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), - Instruction(opname='JUMP_FORWARD', opcode=74, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=83, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=33, argval=194, argrepr='to L8', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=1, argval=1, argrepr='', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=42, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=107, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=6, argval=6, argrepr='', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=54, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=2, argval=176, argrepr='to L6', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=72, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=4, argval=4, argrepr='', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=54, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=2, argval=192, argrepr='to L7', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=72, arg=39, argval=114, argrepr='to L5', offset=188, start_offset=188, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_FORWARD', opcode=74, arg=11, argval=216, argrepr='to L9', offset=192, start_offset=192, starts_line=True, line_number=17, label=7, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=216, start_offset=216, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=1, argval=1, argrepr='', offset=218, start_offset=218, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=89, arg=0, argval=0, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=42, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=81, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=57, arg=1, argval=1, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=90, arg=1, argval=1, argrepr='__exit__', offset=232, start_offset=232, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=112, arg=2, argval=2, argrepr='', offset=234, start_offset=234, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=112, arg=3, argval=3, argrepr='', offset=236, start_offset=236, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=90, arg=0, argval=0, argrepr='__enter__', offset=238, start_offset=238, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=0, argval=0, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=107, arg=1, argval='dodgy', argrepr='dodgy', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=250, start_offset=250, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=260, start_offset=260, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=276, start_offset=276, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=3, argval=3, argrepr='', offset=278, start_offset=278, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=286, start_offset=286, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=288, start_offset=288, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=298, start_offset=298, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=300, start_offset=300, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=308, start_offset=308, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=3, argval=None, argrepr='None', offset=310, start_offset=310, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='WITH_EXCEPT_START', opcode=41, arg=None, argval=None, argrepr='', offset=316, start_offset=316, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=1, argval=332, argrepr='to L11', offset=326, start_offset=326, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='RERAISE', opcode=100, arg=2, argval=2, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=338, start_offset=338, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=73, arg=28, argval=288, argrepr='to L10', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=344, start_offset=344, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=352, start_offset=352, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=14, argval=396, argrepr='to L12', offset=364, start_offset=364, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=370, start_offset=370, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=380, start_offset=380, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=390, start_offset=390, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=392, start_offset=392, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=73, arg=54, argval=288, argrepr='to L10', offset=394, start_offset=394, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=100, arg=0, argval=0, argrepr='', offset=396, start_offset=396, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=398, start_offset=398, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=400, start_offset=400, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=402, start_offset=402, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=404, start_offset=404, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=87, arg=3, argval='print', argrepr='print + NULL', offset=406, start_offset=406, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=79, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=416, start_offset=416, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=418, start_offset=418, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=100, arg=0, argval=0, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=430, start_offset=430, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='FOR_ITER', opcode=68, arg=34, argval=96, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=108, arg=0, argval='i', argrepr='i', offset=30, start_offset=30, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=32, start_offset=32, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=52, start_offset=52, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=54, start_offset=54, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=4, argval=4, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=18, argval='<', argrepr='bool(<)', offset=58, start_offset=58, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=3, argval=72, argrepr='to L2', offset=62, start_offset=62, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=66, start_offset=66, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=24, argval=24, argrepr='to L1', offset=68, start_offset=68, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=72, start_offset=72, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=6, argval=6, argrepr='', offset=74, start_offset=74, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=148, argval='>', argrepr='bool(>)', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=3, argval=90, argrepr='to L3', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=33, argval=24, argrepr='to L1', offset=86, start_offset=86, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=True, line_number=None, label=3, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=92, start_offset=92, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=75, arg=13, argval=122, argrepr='to L5', offset=94, start_offset=94, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=96, start_offset=96, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=98, start_offset=98, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=100, start_offset=100, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=110, start_offset=110, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=120, start_offset=120, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=84, arg=0, argval='i', argrepr='i', offset=122, start_offset=122, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=38, arg=None, argval=None, argrepr='', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=37, argval=210, argrepr='to L8', offset=132, start_offset=132, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=136, start_offset=136, starts_line=False, line_number=11, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=138, start_offset=138, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=150, start_offset=150, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=158, start_offset=158, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=1, argval=1, argrepr='', offset=162, start_offset=162, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=43, arg=23, argval=23, argrepr='-=', offset=164, start_offset=164, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=108, arg=0, argval='i', argrepr='i', offset=168, start_offset=168, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=170, start_offset=170, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=6, argval=6, argrepr='', offset=172, start_offset=172, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=148, argval='>', argrepr='bool(>)', offset=174, start_offset=174, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=3, argval=188, argrepr='to L6', offset=178, start_offset=178, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=182, start_offset=182, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=33, argval=122, argrepr='to L5', offset=184, start_offset=184, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=188, start_offset=188, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=4, argval=4, argrepr='', offset=190, start_offset=190, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=18, argval='<', argrepr='bool(<)', offset=192, start_offset=192, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=3, argval=206, argrepr='to L7', offset=196, start_offset=196, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=200, start_offset=200, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=42, argval=122, argrepr='to L5', offset=202, start_offset=202, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=206, start_offset=206, starts_line=True, line_number=None, label=7, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=75, arg=11, argval=232, argrepr='to L9', offset=208, start_offset=208, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=210, start_offset=210, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=222, start_offset=222, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=232, start_offset=232, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=1, argval=1, argrepr='', offset=234, start_offset=234, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=0, argval=0, argrepr='', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=43, arg=11, argval=11, argrepr='/', offset=238, start_offset=238, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=242, start_offset=242, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=244, start_offset=244, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=91, arg=1, argval=1, argrepr='__exit__', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=113, arg=2, argval=2, argrepr='', offset=250, start_offset=250, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=113, arg=3, argval=3, argrepr='', offset=252, start_offset=252, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=91, arg=0, argval=0, argrepr='__enter__', offset=254, start_offset=254, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=0, argval=0, argrepr='', offset=256, start_offset=256, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=108, arg=1, argval='dodgy', argrepr='dodgy', offset=264, start_offset=264, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=266, start_offset=266, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=276, start_offset=276, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=278, start_offset=278, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=286, start_offset=286, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=288, start_offset=288, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=290, start_offset=290, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=292, start_offset=292, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=3, argval=3, argrepr='', offset=294, start_offset=294, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=302, start_offset=302, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=304, start_offset=304, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=314, start_offset=314, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=316, start_offset=316, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=324, start_offset=324, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=326, start_offset=326, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=31, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=42, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=38, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=2, argval=350, argrepr='to L11', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=2, argval=2, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=352, start_offset=352, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=354, start_offset=354, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=74, arg=29, argval=304, argrepr='to L10', offset=360, start_offset=360, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=362, start_offset=362, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=31, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=370, start_offset=370, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=15, argval=416, argrepr='to L12', offset=382, start_offset=382, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=388, start_offset=388, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=390, start_offset=390, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=400, start_offset=400, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=402, start_offset=402, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=410, start_offset=410, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=74, arg=56, argval=304, argrepr='to L10', offset=414, start_offset=414, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=0, argval=0, argrepr='', offset=416, start_offset=416, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=418, start_offset=418, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=31, arg=None, argval=None, argrepr='', offset=424, start_offset=424, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=426, start_offset=426, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=436, start_offset=436, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=438, start_offset=438, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=446, start_offset=446, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=0, argval=0, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=450, start_offset=450, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=452, start_offset=452, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=454, start_offset=454, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=79, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), - Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=4, start_offset=4, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=4, start_offset=4, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), ] diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 5a4bcebedf19de..087ac8d456b843 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1491,7 +1491,15 @@ class BranchRecorder(JumpRecorder): event_type = E.BRANCH name = "branch" +class BranchRightRecorder(JumpRecorder): + event_type = E.BRANCH_RIGHT + name = "branch right" + +class BranchLeftRecorder(JumpRecorder): + + event_type = E.BRANCH_LEFT + name = "branch left" class JumpOffsetRecorder: @@ -1504,16 +1512,23 @@ def __init__(self, events, offsets=False): def __call__(self, code, from_, to): self.events.append((self.name, code.co_name, from_, to)) -class BranchOffsetRecorder(JumpOffsetRecorder): +class BranchLeftOffsetRecorder(JumpOffsetRecorder): - event_type = E.BRANCH - name = "branch" + event_type = E.BRANCH_LEFT + name = "branch left" + +class BranchRightOffsetRecorder(JumpOffsetRecorder): + + event_type = E.BRANCH_RIGHT + name = "branch right" JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder JUMP_BRANCH_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder FLOW_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder, ExceptionRecorder, ReturnRecorder -BRANCH_OFFSET_RECORDERS = BranchOffsetRecorder, + +BRANCHES_RECORDERS = BranchLeftRecorder, BranchRightRecorder +BRANCH_OFFSET_RECORDERS = BranchLeftOffsetRecorder, BranchRightOffsetRecorder class TestBranchAndJumpEvents(CheckEvents): maxDiff = None @@ -1529,6 +1544,11 @@ def func(): x = 6 7 + def whilefunc(n=0): + while n < 3: + n += 1 # line 2 + 3 + self.check_events(func, recorders = JUMP_AND_BRANCH_RECORDERS, expected = [ ('branch', 'func', 2, 2), ('branch', 'func', 3, 6), @@ -1558,6 +1578,26 @@ def func(): ('line', 'func', 7), ('line', 'get_events', 11)]) + self.check_events(func, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'func', 2, 2), + ('branch right', 'func', 3, 6), + ('branch left', 'func', 2, 2), + ('branch left', 'func', 3, 4), + ('branch right', 'func', 2, 7)]) + + self.check_events(whilefunc, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'whilefunc', 1, 2), + ('branch left', 'whilefunc', 1, 2), + ('branch left', 'whilefunc', 1, 2), + ('branch right', 'whilefunc', 1, 3)]) + + self.check_events(func, recorders = BRANCH_OFFSET_RECORDERS, expected = [ + ('branch left', 'func', 28, 34), + ('branch right', 'func', 46, 60), + ('branch left', 'func', 28, 34), + ('branch left', 'func', 46, 52), + ('branch right', 'func', 28, 72)]) + def test_except_star(self): class Foo: @@ -1583,8 +1623,8 @@ def func(): ('branch', 'func', 4, 4), ('line', 'func', 5), ('line', 'meth', 1), - ('jump', 'func', 5, '[offset=118]'), - ('branch', 'func', '[offset=122]', '[offset=126]'), + ('jump', 'func', 5, '[offset=120]'), + ('branch', 'func', '[offset=124]', '[offset=130]'), ('line', 'get_events', 11)]) self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ @@ -1598,8 +1638,8 @@ def func(): ('line', 'func', 5), ('line', 'meth', 1), ('return', 'meth', None), - ('jump', 'func', 5, '[offset=118]'), - ('branch', 'func', '[offset=122]', '[offset=126]'), + ('jump', 'func', 5, '[offset=120]'), + ('branch', 'func', '[offset=124]', '[offset=130]'), ('return', 'func', None), ('line', 'get_events', 11)]) @@ -1611,8 +1651,8 @@ def foo(n=0): n += 1 return None - in_loop = ('branch', 'foo', 10, 14) - exit_loop = ('branch', 'foo', 10, 30) + in_loop = ('branch left', 'foo', 10, 16) + exit_loop = ('branch right', 'foo', 10, 32) self.check_events(foo, recorders = BRANCH_OFFSET_RECORDERS, expected = [ in_loop, in_loop, @@ -1852,6 +1892,10 @@ def test_local(self): code = f1.__code__ sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + sys.monitoring.set_local_events(TEST_TOOL, code, E.BRANCH) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.BRANCH_LEFT | E.BRANCH_RIGHT) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) sys.monitoring.set_local_events(TEST_TOOL2, code, E.PY_START) self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), E.PY_START) sys.monitoring.set_local_events(TEST_TOOL, code, 0) @@ -2053,7 +2097,8 @@ def setUp(self): ( 1, E.PY_RETURN, capi.fire_event_py_return, 20), ( 2, E.CALL, capi.fire_event_call, callable, 40), ( 1, E.JUMP, capi.fire_event_jump, 60), - ( 1, E.BRANCH, capi.fire_event_branch, 70), + ( 1, E.BRANCH_RIGHT, capi.fire_event_branch_right, 70), + ( 1, E.BRANCH_LEFT, capi.fire_event_branch_left, 80), ( 1, E.PY_THROW, capi.fire_event_py_throw, ValueError(1)), ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)), ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)), diff --git a/Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst b/Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst new file mode 100644 index 00000000000000..6cd13572ff1893 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-13-14-21-04.gh-issue-122548.hq3Vud.rst @@ -0,0 +1,4 @@ +Adds two new local events to sys.monitoring, ``BRANCH_LEFT`` and +``BRANCH_RIGHT``. This allows the two arms of the branch to be disabled +independently, which should hugely improve performance of branch-level +coverage tools. The old branch event, ``BRANCH`` is now deprecated. diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c index 6fd4a405688f48..e475e3b5937199 100644 --- a/Modules/_testcapi/monitoring.c +++ b/Modules/_testcapi/monitoring.c @@ -286,7 +286,7 @@ fire_event_jump(PyObject *self, PyObject *args) } static PyObject * -fire_event_branch(PyObject *self, PyObject *args) +fire_event_branch_right(PyObject *self, PyObject *args) { PyObject *codelike; int offset; @@ -299,7 +299,25 @@ fire_event_branch(PyObject *self, PyObject *args) if (state == NULL) { return NULL; } - int res = PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset); + int res = PyMonitoring_FireBranchRightEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_branch_left(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireBranchLeftEvent(state, codelike, offset, target_offset); RETURN_INT(teardown_fire(res, state, exception)); } @@ -478,7 +496,8 @@ static PyMethodDef TestMethods[] = { {"fire_event_call", fire_event_call, METH_VARARGS}, {"fire_event_line", fire_event_line, METH_VARARGS}, {"fire_event_jump", fire_event_jump, METH_VARARGS}, - {"fire_event_branch", fire_event_branch, METH_VARARGS}, + {"fire_event_branch_left", fire_event_branch_left, METH_VARARGS}, + {"fire_event_branch_right", fire_event_branch_right, METH_VARARGS}, {"fire_event_py_throw", fire_event_py_throw, METH_VARARGS}, {"fire_event_raise", fire_event_raise, METH_VARARGS}, {"fire_event_c_raise", fire_event_c_raise, METH_VARARGS}, diff --git a/Objects/codeobject.c b/Objects/codeobject.c index eb8de136ee6432..ae232cae86799b 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2197,6 +2197,12 @@ code_linesiterator(PyObject *self, PyObject *Py_UNUSED(args)) return (PyObject *)new_linesiterator(code); } +static PyObject * +code_branchesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) +{ + return _PyInstrumentation_BranchesIterator(code); +} + /*[clinic input] @text_signature "($self, /, **changes)" code.replace @@ -2337,6 +2343,7 @@ code__varname_from_oparg_impl(PyCodeObject *self, int oparg) static struct PyMethodDef code_methods[] = { {"__sizeof__", code_sizeof, METH_NOARGS}, {"co_lines", code_linesiterator, METH_NOARGS}, + {"co_branches", (PyCFunction)code_branchesiterator, METH_NOARGS}, {"co_positions", code_positionsiterator, METH_NOARGS}, CODE_REPLACE_METHODDEF CODE__VARNAME_FROM_OPARG_METHODDEF diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 99b0fa48e01c8b..a0007830e8cbc0 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,37 +1,37 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, - 0,0,0,0,0,243,168,0,0,0,149,0,89,0,79,0, - 70,0,111,0,89,0,79,0,70,1,111,1,88,2,31,0, - 79,1,49,1,0,0,0,0,0,0,29,0,88,2,31,0, - 79,2,88,0,77,6,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,49,2,0,0,0,0,0,0, - 29,0,88,1,77,8,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,31,0,49,0,0,0,0,0, - 0,0,79,3,2,0,0,0,111,5,79,4,16,0,67,20, - 0,0,111,6,88,2,31,0,79,5,88,6,12,0,79,6, - 88,5,88,6,2,0,0,0,12,0,47,4,49,1,0,0, - 0,0,0,0,29,0,72,22,0,0,9,0,29,0,79,0, - 33,0,41,7,78,122,18,70,114,111,122,101,110,32,72,101, - 108,108,111,32,87,111,114,108,100,122,8,115,121,115,46,97, - 114,103,118,218,6,99,111,110,102,105,103,41,5,218,12,112, - 114,111,103,114,97,109,95,110,97,109,101,218,10,101,120,101, - 99,117,116,97,98,108,101,218,15,117,115,101,95,101,110,118, - 105,114,111,110,109,101,110,116,218,17,99,111,110,102,105,103, - 117,114,101,95,99,95,115,116,100,105,111,218,14,98,117,102, - 102,101,114,101,100,95,115,116,100,105,111,122,7,99,111,110, - 102,105,103,32,122,2,58,32,41,7,218,3,115,121,115,218, - 17,95,116,101,115,116,105,110,116,101,114,110,97,108,99,97, - 112,105,218,5,112,114,105,110,116,218,4,97,114,103,118,218, - 11,103,101,116,95,99,111,110,102,105,103,115,114,2,0,0, - 0,218,3,107,101,121,169,0,243,0,0,0,0,218,18,116, - 101,115,116,95,102,114,111,122,101,110,109,97,105,110,46,112, - 121,218,8,60,109,111,100,117,108,101,62,114,17,0,0,0, - 1,0,0,0,115,94,0,0,0,240,3,1,1,1,243,8, - 0,1,11,219,0,24,225,0,5,208,6,26,212,0,27,217, - 0,5,128,106,144,35,151,40,145,40,212,0,27,216,9,26, - 215,9,38,210,9,38,211,9,40,168,24,209,9,50,128,6, - 243,2,6,12,2,128,67,241,14,0,5,10,136,71,144,67, - 144,53,152,2,152,54,160,35,153,59,152,45,208,10,40,214, - 4,41,243,15,6,12,2,114,15,0,0,0, + 0,0,0,0,0,243,170,0,0,0,149,0,90,0,80,0, + 71,0,112,0,90,0,80,0,71,1,112,1,89,2,32,0, + 80,1,50,1,0,0,0,0,0,0,30,0,89,2,32,0, + 80,2,89,0,78,6,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,50,2,0,0,0,0,0,0, + 30,0,89,1,78,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,32,0,50,0,0,0,0,0, + 0,0,80,3,2,0,0,0,112,5,80,4,16,0,68,21, + 0,0,28,0,112,6,89,2,32,0,80,5,89,6,12,0, + 80,6,89,5,89,6,2,0,0,0,12,0,48,4,50,1, + 0,0,0,0,0,0,30,0,73,23,0,0,9,0,30,0, + 80,0,34,0,41,7,78,122,18,70,114,111,122,101,110,32, + 72,101,108,108,111,32,87,111,114,108,100,122,8,115,121,115, + 46,97,114,103,118,218,6,99,111,110,102,105,103,41,5,218, + 12,112,114,111,103,114,97,109,95,110,97,109,101,218,10,101, + 120,101,99,117,116,97,98,108,101,218,15,117,115,101,95,101, + 110,118,105,114,111,110,109,101,110,116,218,17,99,111,110,102, + 105,103,117,114,101,95,99,95,115,116,100,105,111,218,14,98, + 117,102,102,101,114,101,100,95,115,116,100,105,111,122,7,99, + 111,110,102,105,103,32,122,2,58,32,41,7,218,3,115,121, + 115,218,17,95,116,101,115,116,105,110,116,101,114,110,97,108, + 99,97,112,105,218,5,112,114,105,110,116,218,4,97,114,103, + 118,218,11,103,101,116,95,99,111,110,102,105,103,115,114,2, + 0,0,0,218,3,107,101,121,169,0,243,0,0,0,0,218, + 18,116,101,115,116,95,102,114,111,122,101,110,109,97,105,110, + 46,112,121,218,8,60,109,111,100,117,108,101,62,114,17,0, + 0,0,1,0,0,0,115,94,0,0,0,240,3,1,1,1, + 243,8,0,1,11,219,0,24,225,0,5,208,6,26,212,0, + 27,217,0,5,128,106,144,35,151,40,145,40,212,0,27,216, + 9,26,215,9,38,210,9,38,211,9,40,168,24,209,9,50, + 128,6,244,2,6,12,2,128,67,241,14,0,5,10,136,71, + 144,67,144,53,152,2,152,54,160,35,153,59,152,45,208,10, + 40,214,4,41,243,15,6,12,2,114,15,0,0,0, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b67264f0440869..cf089c368b651c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -148,6 +148,8 @@ dummy_func( RESUME_CHECK, }; + macro(NOT_TAKEN) = NOP; + op(_CHECK_PERIODIC, (--)) { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); @@ -2723,7 +2725,7 @@ dummy_func( int flag = PyStackRef_IsFalse(cond); DEAD(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } replaced op(_POP_JUMP_IF_TRUE, (cond -- )) { @@ -2731,7 +2733,7 @@ dummy_func( int flag = PyStackRef_IsTrue(cond); DEAD(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } op(_IS_NONE, (value -- b)) { @@ -2923,13 +2925,11 @@ dummy_func( macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { - _Py_CODEUNIT *target; _PyStackRef iter_stackref = TOP(); PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next != NULL) { PUSH(PyStackRef_FromPyObjectSteal(next)); - target = next_instr; } else { if (_PyErr_Occurred(tstate)) { @@ -2946,9 +2946,9 @@ dummy_func( STACK_SHRINK(1); PyStackRef_CLOSE(iter_stackref); /* Skip END_FOR and POP_TOP */ - target = next_instr + oparg + 2; + _Py_CODEUNIT *target = next_instr + oparg + 2; + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); } - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); } op(_ITER_CHECK_LIST, (iter -- iter)) { @@ -4736,6 +4736,10 @@ dummy_func( INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); } + inst(INSTRUMENTED_NOT_TAKEN, ( -- )) { + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + } + macro(INSTRUMENTED_JUMP_BACKWARD) = unused/1 + _CHECK_PERIODIC + @@ -4744,51 +4748,43 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { _PyStackRef cond = POP(); assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - int offset = flag * oparg; - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { _PyStackRef cond = POP(); assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - int offset = flag * oparg; - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } } inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { _PyStackRef value_stackref = POP(); - int flag = PyStackRef_IsNone(value_stackref); - int offset; - if (flag) { - offset = oparg; + int jump = PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { PyStackRef_CLOSE(value_stackref); - offset = 0; } - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { _PyStackRef value_stackref = POP(); - int offset; - int nflag = PyStackRef_IsNone(value_stackref); - if (nflag) { - offset = 0; - } - else { + int jump = !PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { PyStackRef_CLOSE(value_stackref); - offset = oparg; + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } tier1 inst(EXTENDED_ARG, ( -- )) { diff --git a/Python/codegen.c b/Python/codegen.c index a5e550cf8c947e..6d3272edfdbf94 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -406,7 +406,13 @@ codegen_addop_j(instr_sequence *seq, location loc, assert(IS_JUMP_TARGET_LABEL(target)); assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyInstructionSequence_Addop(seq, opcode, target.id, loc); + if (_PyInstructionSequence_Addop(seq, opcode, target.id, loc) != SUCCESS) { + return ERROR; + } + if (IS_CONDITIONAL_JUMP_OPCODE(opcode) || opcode == FOR_ITER) { + return _PyInstructionSequence_Addop(seq, NOT_TAKEN, 0, NO_LOCATION); + } + return SUCCESS; } #define ADDOP_JUMP(C, LOC, OP, O) \ @@ -682,7 +688,6 @@ codegen_setup_annotations_scope(compiler *c, location loc, ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]); NEW_JUMP_TARGET_LABEL(c, body); ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); - ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); ADDOP_I(c, loc, RAISE_VARARGS, 1); USE_LABEL(c, body); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index de61a64a6e3374..9bfc3e7f0b112b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5668,6 +5668,8 @@ /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + /* _INSTRUMENTED_NOT_TAKEN is not a viable micro-op for tier 2 because it is instrumented */ + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */ /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is instrumented */ diff --git a/Python/flowgraph.c b/Python/flowgraph.c index b1097b64469ecd..64df6290de06ba 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -557,6 +557,12 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } + assert(b->b_next->b_iused > 0); + assert(b->b_next->b_instr[0].i_opcode == NOT_TAKEN); + b->b_next->b_instr[0].i_opcode = NOP; + b->b_next->b_instr[0].i_loc = NO_LOCATION; + RETURN_IF_ERROR( + basicblock_addop(backwards_jump, NOT_TAKEN, 0, last->i_loc)); RETURN_IF_ERROR( basicblock_add_jump(backwards_jump, JUMP, target, last->i_loc)); last->i_opcode = reversed_opcode; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8a89ba890fd9c9..ac89891df80590 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4618,7 +4618,6 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ - _Py_CODEUNIT *target; _PyStackRef iter_stackref = TOP(); PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4626,7 +4625,6 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (next != NULL) { PUSH(PyStackRef_FromPyObjectSteal(next)); - target = next_instr; } else { if (_PyErr_Occurred(tstate)) { @@ -4647,9 +4645,9 @@ STACK_SHRINK(1); PyStackRef_CLOSE(iter_stackref); /* Skip END_FOR and POP_TOP */ - target = next_instr + oparg + 2; + _Py_CODEUNIT *target = next_instr + oparg + 2; + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); } - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -4754,6 +4752,15 @@ GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } + TARGET(INSTRUMENTED_NOT_TAKEN) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + DISPATCH(); + } + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4762,10 +4769,11 @@ /* Skip 1 cache entry */ _PyStackRef cond = POP(); assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - int offset = flag * oparg; - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } DISPATCH(); } @@ -4776,17 +4784,14 @@ INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ _PyStackRef value_stackref = POP(); - int flag = PyStackRef_IsNone(value_stackref); - int offset; - if (flag) { - offset = oparg; + int jump = PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { PyStackRef_CLOSE(value_stackref); - offset = 0; } - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -4797,19 +4802,12 @@ INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ _PyStackRef value_stackref = POP(); - int offset; - int nflag = PyStackRef_IsNone(value_stackref); - if (nflag) { - offset = 0; - } - else { + int jump = !PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { PyStackRef_CLOSE(value_stackref); - offset = oparg; + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -4821,10 +4819,11 @@ /* Skip 1 cache entry */ _PyStackRef cond = POP(); assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - int offset = flag * oparg; - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } DISPATCH(); } @@ -6659,6 +6658,13 @@ DISPATCH(); } + TARGET(NOT_TAKEN) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + DISPATCH(); + } + TARGET(POP_EXCEPT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -6687,7 +6693,7 @@ assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6719,7 +6725,7 @@ assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -6752,7 +6758,7 @@ assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -6770,7 +6776,7 @@ assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(oparg * flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3503809e3306cb..e4255bfad8c41a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -85,22 +85,24 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, [INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, [INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [FOR_ITER] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [FOR_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT, [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, [INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, [END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, [INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, + [NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT, + [INSTRUMENTED_NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT, }; static const uint8_t DE_INSTRUMENT[256] = { @@ -120,6 +122,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_END_FOR] = END_FOR, [INSTRUMENTED_END_SEND] = END_SEND, [INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = NOT_TAKEN, }; static const uint8_t INSTRUMENTED_OPCODES[256] = { @@ -155,6 +158,8 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, [LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, + [NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, @@ -323,33 +328,8 @@ _PyInstruction_GetLength(PyCodeObject *code, int offset) { ASSERT_WORLD_STOPPED_OR_LOCKED(code); - int opcode = - FT_ATOMIC_LOAD_UINT8_RELAXED(_PyCode_CODE(code)[offset].op.code); - assert(opcode != 0); - assert(opcode != RESERVED); - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[offset].original_opcode; - } - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - opcode = deinstrumented; - } - else { - opcode = _PyOpcode_Deopt[opcode]; - } - assert(opcode != 0); - if (opcode == ENTER_EXECUTOR) { - int exec_index = _PyCode_CODE(code)[offset].op.arg; - _PyExecutorObject *exec = code->co_executors->executors[exec_index]; - opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; - } - assert(!is_instrumented(opcode)); - assert(opcode != ENTER_EXECUTOR); - assert(opcode == _PyOpcode_Deopt[opcode]); - return 1 + _PyOpcode_Caches[opcode]; + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(code, offset); + return 1 + _PyOpcode_Caches[inst.op.code]; } #ifdef INSTRUMENT_DEBUG @@ -599,16 +579,15 @@ _Py_GetBaseCodeUnit(PyCodeObject *code, int i) int opcode = inst.op.code; if (opcode < MIN_INSTRUMENTED_OPCODE) { inst.op.code = _PyOpcode_Deopt[opcode]; - assert(inst.op.code <= RESUME); + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); return inst; } if (opcode == ENTER_EXECUTOR) { _PyExecutorObject *exec = code->co_executors->executors[inst.op.arg]; opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; inst.op.code = opcode; - assert(opcode <= RESUME); inst.op.arg = exec->vm_data.oparg; - assert(inst.op.code <= RESUME); + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); return inst; } if (opcode == INSTRUMENTED_LINE) { @@ -1084,6 +1063,8 @@ static const char *const event_names [] = { [PY_MONITORING_EVENT_INSTRUCTION] = "INSTRUCTION", [PY_MONITORING_EVENT_JUMP] = "JUMP", [PY_MONITORING_EVENT_BRANCH] = "BRANCH", + [PY_MONITORING_EVENT_BRANCH_LEFT] = "BRANCH_LEFT", + [PY_MONITORING_EVENT_BRANCH_RIGHT] = "BRANCH_RIGHT", [PY_MONITORING_EVENT_C_RETURN] = "C_RETURN", [PY_MONITORING_EVENT_PY_THROW] = "PY_THROW", [PY_MONITORING_EVENT_RAISE] = "RAISE", @@ -1111,6 +1092,10 @@ call_instrumentation_vector( /* Offset visible to user should be the offset in bytes, as that is the * convention for APIs involving code offsets. */ int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); + if (event == PY_MONITORING_EVENT_BRANCH_LEFT) { + assert(EVENT_FOR_OPCODE[_Py_GetBaseCodeUnit(code, offset-2).op.code] == PY_MONITORING_EVENT_BRANCH_RIGHT); + bytes_offset -= 4; + } PyObject *offset_obj = PyLong_FromLong(bytes_offset); if (offset_obj == NULL) { return -1; @@ -1191,7 +1176,8 @@ _Py_call_instrumentation_jump( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target) { assert(event == PY_MONITORING_EVENT_JUMP || - event == PY_MONITORING_EVENT_BRANCH); + event == PY_MONITORING_EVENT_BRANCH_RIGHT || + event == PY_MONITORING_EVENT_BRANCH_LEFT); assert(frame->instr_ptr == instr); int to = (int)(target - _PyFrame_GetBytecode(frame)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); @@ -1427,19 +1413,6 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* return next_opcode; } - -PyObject * -_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) -{ - PyInterpreterState *is = _PyInterpreterState_GET(); - assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS); - PyObject *callback = _Py_atomic_exchange_ptr(&is->monitoring_callables[tool_id][event_id], - Py_XNewRef(obj)); - - return callback; -} - static void initialize_tools(PyCodeObject *code) { @@ -2312,6 +2285,10 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) return NULL; } event_set &= ~C_RETURN_EVENTS; + if (event_set & (1 << PY_MONITORING_EVENT_BRANCH)) { + event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); + event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); + } if (_PyMonitoring_SetEvents(tool_id, event_set)) { return NULL; } @@ -2384,6 +2361,10 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, return NULL; } event_set &= ~C_RETURN_EVENTS; + if (event_set & (1 << PY_MONITORING_EVENT_BRANCH)) { + event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); + event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); + } if (event_set < 0 || event_set >= (1 << _PY_MONITORING_LOCAL_EVENTS)) { PyErr_Format(PyExc_ValueError, "invalid local event set 0x%x", event_set); return NULL; @@ -2711,7 +2692,27 @@ _PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int3 assert(state->active); PyObject *args[4] = { NULL, NULL, NULL, target_offset }; return capi_call_instrumentation(state, codelike, offset, args, 3, - PY_MONITORING_EVENT_BRANCH); + PY_MONITORING_EVENT_BRANCH_RIGHT); +} + +int +_PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_BRANCH_RIGHT); +} + +int +_PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_BRANCH_LEFT); } int @@ -2849,3 +2850,213 @@ _PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelik Py_DECREF(exc); return exception_event_teardown(err, NULL); } + + + +/* Handle legacy BRANCH event */ + +typedef struct _PyLegacyBranchEventHandler { + PyObject_HEAD + vectorcallfunc vectorcall; + PyObject *handler; + bool right; + int tool_id; +} _PyLegacyBranchEventHandler; + +static void +dealloc_branch_handler(_PyLegacyBranchEventHandler *self) +{ + Py_CLEAR(self->handler); + PyObject_Free((PyObject *)self); +} + +static PyTypeObject _PyLegacyBranchEventHandler_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "sys.monitoring.branch_event_handler", + sizeof(_PyLegacyBranchEventHandler), + .tp_dealloc = (destructor)dealloc_branch_handler, + .tp_vectorcall_offset = offsetof(_PyLegacyBranchEventHandler, vectorcall), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_call = PyVectorcall_Call, +}; + + +static PyObject * +branch_handler( + _PyLegacyBranchEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + PyObject *res = PyObject_Vectorcall(self->handler, args, nargsf, kwnames); + if (res == &_PyInstrumentation_DISABLE) { + // Find the other instrumented instruction and remove tool + assert(PyVectorcall_NARGS(nargsf) >= 2); + PyObject *offset_obj = args[1]; + int bytes_offset = PyLong_AsLong(offset_obj); + if (PyErr_Occurred()) { + return NULL; + } + PyCodeObject *code = (PyCodeObject *)args[0]; + if (!PyCode_Check(code) || (bytes_offset & 1)) { + return res; + } + int offset = bytes_offset / 2; + /* We need FOR_ITER and POP_JUMP_ to be the same size */ + assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1); + if (self->right) { + offset += 2; + } + if (offset >= Py_SIZE(code)) { + return res; + } + int other_event = self->right ? + PY_MONITORING_EVENT_BRANCH_LEFT : PY_MONITORING_EVENT_BRANCH_RIGHT; + LOCK_CODE(code); + remove_tools(code, offset, other_event, 1 << self->tool_id); + UNLOCK_CODE(); + } + return res; +} + +static PyObject *make_branch_handler(int tool_id, PyObject *handler, bool right) +{ + _PyLegacyBranchEventHandler *callback = + PyObject_NEW(_PyLegacyBranchEventHandler, &_PyLegacyBranchEventHandler_Type); + if (callback == NULL) { + return NULL; + } + callback->vectorcall = (vectorcallfunc)branch_handler; + callback->handler = Py_NewRef(handler); + callback->right = right; + callback->tool_id = tool_id; + return (PyObject *)callback; +} + +/* Consumes a reference to obj */ +static PyObject *exchange_callables(int tool_id, int event_id, PyObject *obj) +{ + PyInterpreterState *is = _PyInterpreterState_GET(); + return _Py_atomic_exchange_ptr(&is->monitoring_callables[tool_id][event_id], obj); +} + +PyObject * +_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS); + PyObject *res; + if (event_id == PY_MONITORING_EVENT_BRANCH) { + PyObject *left, *right; + if (obj == NULL) { + left = NULL; + right = NULL; + } + else { + right = make_branch_handler(tool_id, obj, true); + if (right == NULL) { + return NULL; + } + left = make_branch_handler(tool_id, obj, false); + if (left == NULL) { + Py_DECREF(right); + return NULL; + } + } + Py_XDECREF(exchange_callables(tool_id, PY_MONITORING_EVENT_BRANCH_RIGHT, right)); + res = exchange_callables(tool_id, PY_MONITORING_EVENT_BRANCH_LEFT, left); + } + else { + res = exchange_callables(tool_id, event_id, Py_XNewRef(obj)); + } + if (res != NULL && Py_TYPE(res) == &_PyLegacyBranchEventHandler_Type) { + _PyLegacyBranchEventHandler *wrapper = (_PyLegacyBranchEventHandler *)res; + res = Py_NewRef(wrapper->handler); + Py_DECREF(wrapper); + } + return res; +} + +/* Branch Iterator */ + +typedef struct { + PyObject_HEAD + PyCodeObject *bi_code; + int bi_offset; +} branchesiterator; + +static PyObject * +int_triple(int a, int b, int c) { + PyObject *obja = PyLong_FromLong(a); + PyObject *objb = NULL; + PyObject *objc = NULL; + if (obja == NULL) { + goto error; + } + objb = PyLong_FromLong(b); + if (objb == NULL) { + goto error; + } + objc = PyLong_FromLong(c); + if (objc == NULL) { + goto error; + } + PyObject *array[3] = { obja, objb, objc }; + return _PyTuple_FromArraySteal(array, 3); +error: + Py_XDECREF(obja); + Py_XDECREF(objb); + Py_XDECREF(objc); + return NULL; +} + +static PyObject * +branchesiter_next(branchesiterator *bi) +{ + int offset = bi->bi_offset; + while (offset < Py_SIZE(bi->bi_code)) { + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(bi->bi_code, offset); + int next_offset = offset + _PyInstruction_GetLength(bi->bi_code, offset); + int event = EVENT_FOR_OPCODE[inst.op.code]; + if (event == PY_MONITORING_EVENT_BRANCH_RIGHT) { + /* Skip NOT_TAKEN */ + int not_taken = next_offset + 1; + bi->bi_offset = not_taken; + return int_triple(offset*2, not_taken*2, (next_offset + inst.op.arg)*2); + } + offset = next_offset; + } + return NULL; +} + +static void +branchesiter_dealloc(branchesiterator *bi) +{ + Py_DECREF(bi->bi_code); + PyObject_Free(bi); +} + +static PyTypeObject _PyBranchesIterator = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "line_iterator", /* tp_name */ + sizeof(branchesiterator), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + .tp_dealloc = (destructor)branchesiter_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_iter = PyObject_SelfIter, + .tp_iternext = (iternextfunc)branchesiter_next, + .tp_free = PyObject_Del, +}; + +PyObject * +_PyInstrumentation_BranchesIterator(PyCodeObject *code) +{ + + branchesiterator *bi = (branchesiterator *)PyType_GenericAlloc(&_PyBranchesIterator, 0); + if (bi == NULL) { + return NULL; + } + bi->bi_code = (PyCodeObject*)Py_NewRef(code); + bi->bi_offset = 0; + return (PyObject *)bi; +} diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index c93941dcac4abf..7f3fb9c9a63dd1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -27,6 +27,7 @@ static void *opcode_targets[256] = { &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_NOP, + &&TARGET_NOT_TAKEN, &&TARGET_POP_EXCEPT, &&TARGET_POP_TOP, &&TARGET_PUSH_EXC_INFO, @@ -147,7 +148,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -235,7 +235,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, @@ -244,6 +243,7 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_JUMP_FORWARD, + &&TARGET_INSTRUMENTED_NOT_TAKEN, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 33b34d6fa0d3f9..2c3133d7107df2 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2343,6 +2343,8 @@ /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_NOT_TAKEN is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index badd7b79102310..a1ec1927eb56df 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -106,6 +106,8 @@ Python/context.c - PyContextToken_Type - Python/context.c - PyContextVar_Type - Python/context.c - PyContext_Type - Python/instruction_sequence.c - _PyInstructionSequence_Type - +Python/instrumentation.c - _PyLegacyBranchEventHandler_Type - +Python/instrumentation.c - _PyBranchesIterator - Python/traceback.c - PyTraceBack_Type - ##----------------------- From 1b15c89a17ca3de6b05de5379b8717e9738c51ef Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 19 Dec 2024 10:21:17 -0800 Subject: [PATCH 248/427] gh-115999: Specialize `STORE_ATTR` in free-threaded builds. (gh-127838) * Add `_PyDictKeys_StringLookupSplit` which does locking on dict keys and use in place of `_PyDictKeys_StringLookup`. * Change `_PyObject_TryGetInstanceAttribute` to use that function in the case of split keys. * Add `unicodekeys_lookup_split` helper which allows code sharing between `_Py_dict_lookup` and `_PyDictKeys_StringLookupSplit`. * Fix locking for `STORE_ATTR_INSTANCE_VALUE`. Create `_GUARD_TYPE_VERSION_AND_LOCK` uop so that object stays locked and `tp_version_tag` cannot change. * Pass `tp_version_tag` to `specialize_dict_access()`, ensuring the version we store on the cache is the correct one (in case of it changing during the specalize analysis). * Split `analyze_descriptor` into `analyze_descriptor_load` and `analyze_descriptor_store` since those don't share much logic. Add `descriptor_is_class` helper function. * In `specialize_dict_access`, double check `_PyObject_GetManagedDict()` in case we race and dict was materialized before the lock. * Avoid borrowed references in `_Py_Specialize_StoreAttr()`. * Use `specialize()` and `unspecialize()` helpers. * Add unit tests to ensure specializing happens as expected in FT builds. * Add unit tests to attempt to trigger data races (useful for running under TSAN). * Add `has_split_table` function to `_testinternalcapi`. --- Include/internal/pycore_dict.h | 1 + Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_uop_ids.h | 201 +++++++-------- Include/internal/pycore_uop_metadata.h | 6 +- Lib/test/test_free_threading/test_races.py | 141 +++++++++++ Lib/test/test_opcache.py | 66 +++++ Modules/_testinternalcapi.c | 9 + Objects/dictobject.c | 66 +++-- Python/bytecodes.c | 65 +++-- Python/executor_cases.c.h | 93 +++++-- Python/generated_cases.c.h | 86 ++++--- Python/optimizer_cases.c.h | 4 + Python/specialize.c | 271 +++++++++++++-------- 13 files changed, 716 insertions(+), 297 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 6e4a308226f3fe..71927006d1cd48 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -114,6 +114,7 @@ extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); +extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e09fff062b5202..5fb236836dccd9 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -2141,7 +2141,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -2340,7 +2340,7 @@ _PyOpcode_macro_expansion[256] = { [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } }, [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION_AND_LOCK, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, [STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 5fc57e48f500d0..92515b4230ccb4 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -139,15 +139,16 @@ extern "C" { #define _GUARD_TOS_FLOAT 386 #define _GUARD_TOS_INT 387 #define _GUARD_TYPE_VERSION 388 +#define _GUARD_TYPE_VERSION_AND_LOCK 389 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 389 -#define _INIT_CALL_PY_EXACT_ARGS 390 -#define _INIT_CALL_PY_EXACT_ARGS_0 391 -#define _INIT_CALL_PY_EXACT_ARGS_1 392 -#define _INIT_CALL_PY_EXACT_ARGS_2 393 -#define _INIT_CALL_PY_EXACT_ARGS_3 394 -#define _INIT_CALL_PY_EXACT_ARGS_4 395 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 390 +#define _INIT_CALL_PY_EXACT_ARGS 391 +#define _INIT_CALL_PY_EXACT_ARGS_0 392 +#define _INIT_CALL_PY_EXACT_ARGS_1 393 +#define _INIT_CALL_PY_EXACT_ARGS_2 394 +#define _INIT_CALL_PY_EXACT_ARGS_3 395 +#define _INIT_CALL_PY_EXACT_ARGS_4 396 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -160,143 +161,143 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 396 -#define _IS_NONE 397 +#define _INTERNAL_INCREMENT_OPT_COUNTER 397 +#define _IS_NONE 398 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 398 -#define _ITER_CHECK_RANGE 399 -#define _ITER_CHECK_TUPLE 400 -#define _ITER_JUMP_LIST 401 -#define _ITER_JUMP_RANGE 402 -#define _ITER_JUMP_TUPLE 403 -#define _ITER_NEXT_LIST 404 -#define _ITER_NEXT_RANGE 405 -#define _ITER_NEXT_TUPLE 406 -#define _JUMP_TO_TOP 407 +#define _ITER_CHECK_LIST 399 +#define _ITER_CHECK_RANGE 400 +#define _ITER_CHECK_TUPLE 401 +#define _ITER_JUMP_LIST 402 +#define _ITER_JUMP_RANGE 403 +#define _ITER_JUMP_TUPLE 404 +#define _ITER_NEXT_LIST 405 +#define _ITER_NEXT_RANGE 406 +#define _ITER_NEXT_TUPLE 407 +#define _JUMP_TO_TOP 408 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 408 -#define _LOAD_ATTR_CLASS 409 -#define _LOAD_ATTR_CLASS_0 410 -#define _LOAD_ATTR_CLASS_1 411 +#define _LOAD_ATTR 409 +#define _LOAD_ATTR_CLASS 410 +#define _LOAD_ATTR_CLASS_0 411 +#define _LOAD_ATTR_CLASS_1 412 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 412 -#define _LOAD_ATTR_INSTANCE_VALUE_0 413 -#define _LOAD_ATTR_INSTANCE_VALUE_1 414 -#define _LOAD_ATTR_METHOD_LAZY_DICT 415 -#define _LOAD_ATTR_METHOD_NO_DICT 416 -#define _LOAD_ATTR_METHOD_WITH_VALUES 417 -#define _LOAD_ATTR_MODULE 418 -#define _LOAD_ATTR_MODULE_FROM_KEYS 419 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 420 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 421 -#define _LOAD_ATTR_PROPERTY_FRAME 422 -#define _LOAD_ATTR_SLOT 423 -#define _LOAD_ATTR_SLOT_0 424 -#define _LOAD_ATTR_SLOT_1 425 -#define _LOAD_ATTR_WITH_HINT 426 +#define _LOAD_ATTR_INSTANCE_VALUE 413 +#define _LOAD_ATTR_INSTANCE_VALUE_0 414 +#define _LOAD_ATTR_INSTANCE_VALUE_1 415 +#define _LOAD_ATTR_METHOD_LAZY_DICT 416 +#define _LOAD_ATTR_METHOD_NO_DICT 417 +#define _LOAD_ATTR_METHOD_WITH_VALUES 418 +#define _LOAD_ATTR_MODULE 419 +#define _LOAD_ATTR_MODULE_FROM_KEYS 420 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 421 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 422 +#define _LOAD_ATTR_PROPERTY_FRAME 423 +#define _LOAD_ATTR_SLOT 424 +#define _LOAD_ATTR_SLOT_0 425 +#define _LOAD_ATTR_SLOT_1 426 +#define _LOAD_ATTR_WITH_HINT 427 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 427 +#define _LOAD_BYTECODE 428 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 428 -#define _LOAD_CONST_INLINE_BORROW 429 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 430 -#define _LOAD_CONST_INLINE_WITH_NULL 431 +#define _LOAD_CONST_INLINE 429 +#define _LOAD_CONST_INLINE_BORROW 430 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 431 +#define _LOAD_CONST_INLINE_WITH_NULL 432 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 432 -#define _LOAD_FAST_0 433 -#define _LOAD_FAST_1 434 -#define _LOAD_FAST_2 435 -#define _LOAD_FAST_3 436 -#define _LOAD_FAST_4 437 -#define _LOAD_FAST_5 438 -#define _LOAD_FAST_6 439 -#define _LOAD_FAST_7 440 +#define _LOAD_FAST 433 +#define _LOAD_FAST_0 434 +#define _LOAD_FAST_1 435 +#define _LOAD_FAST_2 436 +#define _LOAD_FAST_3 437 +#define _LOAD_FAST_4 438 +#define _LOAD_FAST_5 439 +#define _LOAD_FAST_6 440 +#define _LOAD_FAST_7 441 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 441 -#define _LOAD_GLOBAL_BUILTINS 442 -#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 443 -#define _LOAD_GLOBAL_MODULE 444 -#define _LOAD_GLOBAL_MODULE_FROM_KEYS 445 +#define _LOAD_GLOBAL 442 +#define _LOAD_GLOBAL_BUILTINS 443 +#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 444 +#define _LOAD_GLOBAL_MODULE 445 +#define _LOAD_GLOBAL_MODULE_FROM_KEYS 446 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 446 -#define _LOAD_SMALL_INT_0 447 -#define _LOAD_SMALL_INT_1 448 -#define _LOAD_SMALL_INT_2 449 -#define _LOAD_SMALL_INT_3 450 +#define _LOAD_SMALL_INT 447 +#define _LOAD_SMALL_INT_0 448 +#define _LOAD_SMALL_INT_1 449 +#define _LOAD_SMALL_INT_2 450 +#define _LOAD_SMALL_INT_3 451 #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 451 +#define _MAKE_CALLARGS_A_TUPLE 452 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 452 +#define _MAKE_WARM 453 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 453 -#define _MAYBE_EXPAND_METHOD_KW 454 -#define _MONITOR_CALL 455 -#define _MONITOR_JUMP_BACKWARD 456 -#define _MONITOR_RESUME 457 +#define _MAYBE_EXPAND_METHOD 454 +#define _MAYBE_EXPAND_METHOD_KW 455 +#define _MONITOR_CALL 456 +#define _MONITOR_JUMP_BACKWARD 457 +#define _MONITOR_RESUME 458 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 458 -#define _POP_JUMP_IF_TRUE 459 +#define _POP_JUMP_IF_FALSE 459 +#define _POP_JUMP_IF_TRUE 460 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 460 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 461 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 461 +#define _PUSH_FRAME 462 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 462 -#define _PY_FRAME_KW 463 -#define _QUICKEN_RESUME 464 -#define _REPLACE_WITH_TRUE 465 +#define _PY_FRAME_GENERAL 463 +#define _PY_FRAME_KW 464 +#define _QUICKEN_RESUME 465 +#define _REPLACE_WITH_TRUE 466 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 466 -#define _SEND 467 -#define _SEND_GEN_FRAME 468 +#define _SAVE_RETURN_OFFSET 467 +#define _SEND 468 +#define _SEND_GEN_FRAME 469 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 469 -#define _STORE_ATTR 470 -#define _STORE_ATTR_INSTANCE_VALUE 471 -#define _STORE_ATTR_SLOT 472 -#define _STORE_ATTR_WITH_HINT 473 +#define _START_EXECUTOR 470 +#define _STORE_ATTR 471 +#define _STORE_ATTR_INSTANCE_VALUE 472 +#define _STORE_ATTR_SLOT 473 +#define _STORE_ATTR_WITH_HINT 474 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 474 -#define _STORE_FAST_0 475 -#define _STORE_FAST_1 476 -#define _STORE_FAST_2 477 -#define _STORE_FAST_3 478 -#define _STORE_FAST_4 479 -#define _STORE_FAST_5 480 -#define _STORE_FAST_6 481 -#define _STORE_FAST_7 482 +#define _STORE_FAST 475 +#define _STORE_FAST_0 476 +#define _STORE_FAST_1 477 +#define _STORE_FAST_2 478 +#define _STORE_FAST_3 479 +#define _STORE_FAST_4 480 +#define _STORE_FAST_5 481 +#define _STORE_FAST_6 482 +#define _STORE_FAST_7 483 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 483 -#define _STORE_SUBSCR 484 +#define _STORE_SLICE 484 +#define _STORE_SUBSCR 485 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 485 -#define _TO_BOOL 486 +#define _TIER2_RESUME_CHECK 486 +#define _TO_BOOL 487 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -306,13 +307,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 487 +#define _UNPACK_SEQUENCE 488 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 487 +#define MAX_UOP_ID 488 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index eadfda472a7270..e71194b116e020 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -148,6 +148,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, + [_GUARD_TYPE_VERSION_AND_LOCK] = HAS_EXIT_FLAG, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, @@ -167,7 +168,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = 0, [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_STORE_ATTR_SLOT] = 0, + [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -428,6 +429,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", [_IMPORT_FROM] = "_IMPORT_FROM", [_IMPORT_NAME] = "_IMPORT_NAME", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", @@ -839,6 +841,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_TYPE_VERSION: return 0; + case _GUARD_TYPE_VERSION_AND_LOCK: + return 0; case _CHECK_MANAGED_OBJECT_HAS_VALUES: return 0; case _LOAD_ATTR_INSTANCE_VALUE_0: diff --git a/Lib/test/test_free_threading/test_races.py b/Lib/test/test_free_threading/test_races.py index 09e1d52e3509f9..69982558a067a5 100644 --- a/Lib/test/test_free_threading/test_races.py +++ b/Lib/test/test_free_threading/test_races.py @@ -4,6 +4,7 @@ import threading import time import unittest +import _testinternalcapi from test.support import threading_helper @@ -129,6 +130,146 @@ def mutate(): # with the cell binding being changed). do_race(access, mutate) + def test_racing_to_bool(self): + + seq = [1] + + class C: + def __bool__(self): + return False + + def access(): + if seq: + return 1 + else: + return 2 + + def mutate(): + nonlocal seq + seq = [1] + time.sleep(0) + seq = C() + time.sleep(0) + + do_race(access, mutate) + + def test_racing_store_attr_slot(self): + class C: + __slots__ = ['x', '__dict__'] + + c = C() + + def set_slot(): + for i in range(10): + c.x = i + time.sleep(0) + + def change_type(): + def set_x(self, x): + pass + + def get_x(self): + pass + + C.x = property(get_x, set_x) + time.sleep(0) + del C.x + time.sleep(0) + + do_race(set_slot, change_type) + + def set_getattribute(): + C.__getattribute__ = lambda self, x: x + time.sleep(0) + del C.__getattribute__ + time.sleep(0) + + do_race(set_slot, set_getattribute) + + def test_racing_store_attr_instance_value(self): + class C: + pass + + c = C() + + def set_value(): + for i in range(100): + c.x = i + + set_value() + + def read(): + x = c.x + + def mutate(): + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + time.sleep(0) + del C.x + time.sleep(0) + + do_race(read, mutate) + + def test_racing_store_attr_with_hint(self): + class C: + pass + + c = C() + for i in range(29): + setattr(c, f"_{i}", None) + + def set_value(): + for i in range(100): + c.x = i + + set_value() + + def read(): + x = c.x + + def mutate(): + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + time.sleep(0) + del C.x + time.sleep(0) + + do_race(read, mutate) + + def make_shared_key_dict(self): + class C: + pass + + a = C() + a.x = 1 + return a.__dict__ + + def test_racing_store_attr_dict(self): + """Test STORE_ATTR with various dictionary types.""" + class C: + pass + + c = C() + + def set_value(): + for i in range(20): + c.x = i + + def mutate(): + nonlocal c + c.x = 1 + self.assertTrue(_testinternalcapi.has_inline_values(c)) + for i in range(30): + setattr(c, f"_{i}", None) + self.assertFalse(_testinternalcapi.has_inline_values(c.__dict__)) + c.__dict__ = self.make_shared_key_dict() + self.assertTrue(_testinternalcapi.has_split_table(c.__dict__)) + c.__dict__[1] = None + self.assertFalse(_testinternalcapi.has_split_table(c.__dict__)) + c = C() + + do_race(set_value, mutate) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 94709e2022550a..ad0b0c487a4118 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1383,6 +1383,72 @@ def send_yield_from(): self.assert_specialized(send_yield_from, "SEND_GEN") self.assert_no_opcode(send_yield_from, "SEND") + @cpython_only + @requires_specialization_ft + def test_store_attr_slot(self): + class C: + __slots__ = ['x'] + + def set_slot(): + c = C() + for i in range(100): + c.x = i + + set_slot() + + self.assert_specialized(set_slot, "STORE_ATTR_SLOT") + self.assert_no_opcode(set_slot, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_slot() + self.assert_no_opcode(set_slot, "STORE_ATTR_SLOT") + + @cpython_only + @requires_specialization_ft + def test_store_attr_instance_value(self): + class C: + pass + + def set_value(): + c = C() + for i in range(100): + c.x = i + + set_value() + + self.assert_specialized(set_value, "STORE_ATTR_INSTANCE_VALUE") + self.assert_no_opcode(set_value, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_value() + self.assert_no_opcode(set_value, "STORE_ATTR_INSTANCE_VALUE") + + @cpython_only + @requires_specialization_ft + def test_store_attr_with_hint(self): + class C: + pass + + c = C() + for i in range(29): + setattr(c, f"_{i}", None) + + def set_value(): + for i in range(100): + c.x = i + + set_value() + + self.assert_specialized(set_value, "STORE_ATTR_WITH_HINT") + self.assert_no_opcode(set_value, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_value() + self.assert_no_opcode(set_value, "STORE_ATTR_WITH_HINT") + @cpython_only @requires_specialization_ft def test_to_bool(self): diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 014f89997f7f60..150d34d168f5e4 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1989,6 +1989,14 @@ has_inline_values(PyObject *self, PyObject *obj) Py_RETURN_FALSE; } +static PyObject * +has_split_table(PyObject *self, PyObject *obj) +{ + if (PyDict_Check(obj) && _PyDict_HasSplitTable((PyDictObject *)obj)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} // Circumvents standard version assignment machinery - use with caution and only on // short-lived heap types @@ -2139,6 +2147,7 @@ static PyMethodDef module_functions[] = { {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, {"has_inline_values", has_inline_values, METH_O}, + {"has_split_table", has_split_table, METH_O}, {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, PyDoc_STR("forcefully assign type->tp_version_tag")}, diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 05c93a3e448181..2a054c3f2ae0ff 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1129,6 +1129,35 @@ dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, P return do_lookup(mp, dk, key, hash, compare_generic); } +#ifdef Py_GIL_DISABLED +static Py_ssize_t +unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, + Py_hash_t hash); +#endif + +static Py_ssize_t +unicodekeys_lookup_split(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + Py_ssize_t ix; + assert(dk->dk_kind == DICT_KEYS_SPLIT); + assert(PyUnicode_CheckExact(key)); + +#ifdef Py_GIL_DISABLED + // A split dictionaries keys can be mutated by other dictionaries + // but if we have a unicode key we can avoid locking the shared + // keys. + ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + LOCK_KEYS(dk); + ix = unicodekeys_lookup_unicode(dk, key, hash); + UNLOCK_KEYS(dk); + } +#else + ix = unicodekeys_lookup_unicode(dk, key, hash); +#endif + return ix; +} + /* Lookup a string in a (all unicode) dict keys. * Returns DKIX_ERROR if key is not a string, * or if the dict keys is not all strings. @@ -1153,13 +1182,24 @@ _PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key) return unicodekeys_lookup_unicode(dk, key, hash); } -#ifdef Py_GIL_DISABLED - -static Py_ssize_t -unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, - Py_hash_t hash); - -#endif +/* Like _PyDictKeys_StringLookup() but only works on split keys. Note + * that in free-threaded builds this locks the keys object as required. + */ +Py_ssize_t +_PyDictKeys_StringLookupSplit(PyDictKeysObject* dk, PyObject *key) +{ + assert(dk->dk_kind == DICT_KEYS_SPLIT); + assert(PyUnicode_CheckExact(key)); + Py_hash_t hash = unicode_get_hash(key); + if (hash == -1) { + hash = PyUnicode_Type.tp_hash(key); + if (hash == -1) { + PyErr_Clear(); + return DKIX_ERROR; + } + } + return unicodekeys_lookup_split(dk, key, hash); +} /* The basic lookup function used by all operations. @@ -1192,15 +1232,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu if (PyUnicode_CheckExact(key)) { #ifdef Py_GIL_DISABLED if (kind == DICT_KEYS_SPLIT) { - // A split dictionaries keys can be mutated by other - // dictionaries but if we have a unicode key we can avoid - // locking the shared keys. - ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); - if (ix == DKIX_KEY_CHANGED) { - LOCK_KEYS(dk); - ix = unicodekeys_lookup_unicode(dk, key, hash); - UNLOCK_KEYS(dk); - } + ix = unicodekeys_lookup_split(dk, key, hash); } else { ix = unicodekeys_lookup_unicode(dk, key, hash); @@ -6967,7 +6999,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); - Py_ssize_t ix = _PyDictKeys_StringLookup(keys, name); + Py_ssize_t ix = _PyDictKeys_StringLookupSplit(keys, name); if (ix == DKIX_EMPTY) { *attr = NULL; return true; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cf089c368b651c..30c12dd4dc9205 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1467,7 +1467,7 @@ dummy_func( }; specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; @@ -1476,7 +1476,7 @@ dummy_func( } OPCODE_DEFERRED_INC(STORE_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_STORE_ATTR, (v, owner --)) { @@ -2129,7 +2129,18 @@ dummy_func( op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - EXIT_IF(tp->tp_version_tag != type_version); + EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version); + } + + op(_GUARD_TYPE_VERSION_AND_LOCK, (type_version/2, owner -- owner)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + EXIT_IF(!LOCK_OBJECT(owner_o)); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + EXIT_IF(true); + } } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { @@ -2336,8 +2347,11 @@ dummy_func( assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - EXIT_IF(_PyObject_GetManagedDict(owner_o)); - EXIT_IF(_PyObject_InlineValues(owner_o)->valid == 0); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + EXIT_IF(true); + } } op(_STORE_ATTR_INSTANCE_VALUE, (offset/1, value, owner --)) { @@ -2347,21 +2361,20 @@ dummy_func( assert(_PyObject_GetManagedDict(owner_o) == NULL); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *old_value = *value_ptr; - *value_ptr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { PyDictValues *values = _PyObject_InlineValues(owner_o); Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } macro(STORE_ATTR_INSTANCE_VALUE) = unused/1 + - _GUARD_TYPE_VERSION + + _GUARD_TYPE_VERSION_AND_LOCK + _GUARD_DORV_NO_DICT + _STORE_ATTR_INSTANCE_VALUE; @@ -2370,16 +2383,34 @@ dummy_func( assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); + DEOPT_IF(!LOCK_OBJECT(dict)); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } + #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys)); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } PyObject *old_value = ep->me_value; - DEOPT_IF(old_value == NULL); + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - ep->me_value = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -2395,10 +2426,12 @@ dummy_func( op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o)); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9bfc3e7f0b112b..6e752c57cd70f3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2574,13 +2574,34 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - if (tp->tp_version_tag != type_version) { + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } break; } + case _GUARD_TYPE_VERSION_AND_LOCK: { + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + break; + } + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { _PyStackRef owner; owner = stack_pointer[-1]; @@ -2910,13 +2931,13 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (_PyObject_InlineValues(owner_o)->valid == 0) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } @@ -2932,15 +2953,14 @@ assert(_PyObject_GetManagedDict(owner_o) == NULL); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *old_value = *value_ptr; - *value_ptr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { PyDictValues *values = _PyObject_InlineValues(owner_o); Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2961,30 +2981,50 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries) { + if (!LOCK_OBJECT(dict)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (!DK_IS_UNICODE(dict->ma_keys)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #endif + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyObject *old_value = ep->me_value; if (old_value == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - ep->me_value = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -3002,10 +3042,15 @@ value = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); Py_XDECREF(old_value); PyStackRef_CLOSE(owner); stack_pointer += -2; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ac89891df80590..ee5c55a832d460 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5319,7 +5319,7 @@ uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -5388,7 +5388,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { @@ -5433,7 +5433,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -5476,7 +5476,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -5512,7 +5512,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { @@ -5611,7 +5611,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -5642,7 +5642,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { @@ -5688,7 +5688,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -5750,7 +5750,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -5787,7 +5787,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { @@ -7314,7 +7314,7 @@ owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; @@ -7325,7 +7325,7 @@ } OPCODE_DEFERRED_INC(STORE_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 3 cache entries */ // _STORE_ATTR @@ -7353,21 +7353,29 @@ _PyStackRef owner; _PyStackRef value; /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION + // _GUARD_TYPE_VERSION_AND_LOCK { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } } // _GUARD_DORV_NO_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(_PyObject_GetManagedDict(owner_o), STORE_ATTR); - DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } } // _STORE_ATTR_INSTANCE_VALUE { @@ -7378,15 +7386,14 @@ assert(_PyObject_GetManagedDict(owner_o) == NULL); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *old_value = *value_ptr; - *value_ptr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { PyDictValues *values = _PyObject_InlineValues(owner_o); Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } stack_pointer += -2; @@ -7408,17 +7415,19 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } @@ -7441,7 +7450,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -7451,18 +7460,35 @@ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), STORE_ATTR); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } PyObject *old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - ep->me_value = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -7815,7 +7841,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); } // _REPLACE_WITH_TRUE { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 2c3133d7107df2..b46079ec8a1992 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1121,6 +1121,10 @@ break; } + case _GUARD_TYPE_VERSION_AND_LOCK: { + break; + } + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { break; } diff --git a/Python/specialize.c b/Python/specialize.c index 6c45320f95db8e..349ed472298945 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -741,8 +741,8 @@ static int function_kind(PyCodeObject *code); #ifndef Py_GIL_DISABLED static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); -#endif static uint32_t type_get_version(PyTypeObject *t, int opcode); +#endif static int specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name) @@ -881,71 +881,142 @@ classify_descriptor(PyObject *descriptor, bool has_getattr) return NON_DESCRIPTOR; } -static DescriptorClassification -analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store) +static bool +descriptor_is_class(PyObject *descriptor, PyObject *name) { + return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) && + (descriptor == _PyType_Lookup(&PyBaseObject_Type, name))); +} + +#ifndef Py_GIL_DISABLED +static DescriptorClassification +analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { bool has_getattr = false; - if (store) { - if (type->tp_setattro != PyObject_GenericSetAttr) { + getattrofunc getattro_slot = type->tp_getattro; + if (getattro_slot == PyObject_GenericGetAttr) { + /* Normal attribute lookup; */ + has_getattr = false; + } + else if (getattro_slot == _Py_slot_tp_getattr_hook || + getattro_slot == _Py_slot_tp_getattro) { + /* One or both of __getattribute__ or __getattr__ may have been + overridden See typeobject.c for why these functions are special. */ + PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__)); + PyInterpreterState *interp = _PyInterpreterState_GET(); + bool has_custom_getattribute = getattribute != NULL && + getattribute != interp->callable_cache.object__getattribute__; + PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__)); + has_getattr = getattr != NULL; + Py_XDECREF(getattr); + if (has_custom_getattribute) { + if (getattro_slot == _Py_slot_tp_getattro && + !has_getattr && + Py_IS_TYPE(getattribute, &PyFunction_Type)) { + *descr = getattribute; + return GETATTRIBUTE_IS_PYTHON_FUNCTION; + } + /* Potentially both __getattr__ and __getattribute__ are set. + Too complicated */ + Py_DECREF(getattribute); *descr = NULL; return GETSET_OVERRIDDEN; } + /* Potentially has __getattr__ but no custom __getattribute__. + Fall through to usual descriptor analysis. + Usual attribute lookup should only be allowed at runtime + if we can guarantee that there is no way an exception can be + raised. This means some specializations, e.g. specializing + for property() isn't safe. + */ + Py_XDECREF(getattribute); } else { - getattrofunc getattro_slot = type->tp_getattro; - if (getattro_slot == PyObject_GenericGetAttr) { - /* Normal attribute lookup; */ - has_getattr = false; - } - else if (getattro_slot == _Py_slot_tp_getattr_hook || - getattro_slot == _Py_slot_tp_getattro) { - /* One or both of __getattribute__ or __getattr__ may have been - overridden See typeobject.c for why these functions are special. */ - PyObject *getattribute = _PyType_Lookup(type, - &_Py_ID(__getattribute__)); - PyInterpreterState *interp = _PyInterpreterState_GET(); - bool has_custom_getattribute = getattribute != NULL && - getattribute != interp->callable_cache.object__getattribute__; - has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL; - if (has_custom_getattribute) { - if (getattro_slot == _Py_slot_tp_getattro && - !has_getattr && - Py_IS_TYPE(getattribute, &PyFunction_Type)) { - *descr = getattribute; - return GETATTRIBUTE_IS_PYTHON_FUNCTION; - } - /* Potentially both __getattr__ and __getattribute__ are set. - Too complicated */ - *descr = NULL; - return GETSET_OVERRIDDEN; - } - /* Potentially has __getattr__ but no custom __getattribute__. - Fall through to usual descriptor analysis. - Usual attribute lookup should only be allowed at runtime - if we can guarantee that there is no way an exception can be - raised. This means some specializations, e.g. specializing - for property() isn't safe. - */ - } - else { - *descr = NULL; - return GETSET_OVERRIDDEN; - } + *descr = NULL; + return GETSET_OVERRIDDEN; } - PyObject *descriptor = _PyType_Lookup(type, name); + PyObject *descriptor = _PyType_LookupRef(type, name); *descr = descriptor; - if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) { - if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) { - return DUNDER_CLASS; - } + if (descriptor_is_class(descriptor, name)) { + return DUNDER_CLASS; } return classify_descriptor(descriptor, has_getattr); } +#endif //!Py_GIL_DISABLED + +static DescriptorClassification +analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) +{ + if (type->tp_setattro != PyObject_GenericSetAttr) { + *descr = NULL; + return GETSET_OVERRIDDEN; + } + PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version); + *descr = descriptor; + if (descriptor_is_class(descriptor, name)) { + return DUNDER_CLASS; + } + return classify_descriptor(descriptor, false); +} + +static int +specialize_dict_access_inline( + PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, + int base_op, int values_op) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; + assert(PyUnicode_CheckExact(name)); + Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name); + assert (index != DKIX_ERROR); + if (index == DKIX_EMPTY) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); + return 0; + } + assert(index >= 0); + char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; + Py_ssize_t offset = value_addr - (char *)owner; + if (offset != (uint16_t)offset) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->index = (uint16_t)offset; + write_u32(cache->version, tp_version); + specialize(instr, values_op); + return 1; +} + +static int +specialize_dict_access_hint( + PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, + int base_op, int hint_op) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + // We found an instance with a __dict__. + if (_PyDict_HasSplitTable(dict)) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); + return 0; + } + Py_ssize_t index = _PyDict_LookupIndex(dict, name); + if (index != (uint16_t)index) { + SPECIALIZATION_FAIL(base_op, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_NOT_IN_DICT : + SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->index = (uint16_t)index; + write_u32(cache->version, tp_version); + specialize(instr, hint_op); + return 1; +} + static int specialize_dict_access( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, int base_op, int values_op, int hint_op) { assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT || @@ -956,29 +1027,25 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } - _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && - _PyObject_InlineValues(owner)->valid && + FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner)->valid) && !(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL)) { - PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; - assert(PyUnicode_CheckExact(name)); - Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); - assert (index != DKIX_ERROR); - if (index == DKIX_EMPTY) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); - return 0; + int res; + Py_BEGIN_CRITICAL_SECTION(owner); + PyDictObject *dict = _PyObject_GetManagedDict(owner); + if (dict == NULL) { + // managed dict, not materialized, inline values valid + res = specialize_dict_access_inline(owner, instr, type, kind, name, + tp_version, base_op, values_op); } - assert(index >= 0); - char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; - Py_ssize_t offset = value_addr - (char *)owner; - if (offset != (uint16_t)offset) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); - return 0; + else { + // lost race and dict was created, fail specialization + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER); + res = 0; } - write_u32(cache->version, type->tp_version_tag); - cache->index = (uint16_t)offset; - specialize(instr, values_op); + Py_END_CRITICAL_SECTION(); + return res; } else { PyDictObject *dict = _PyObject_GetManagedDict(owner); @@ -986,25 +1053,14 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; } - // We found an instance with a __dict__. - if (dict->ma_values) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); - return 0; - } - Py_ssize_t index = - _PyDict_LookupIndex(dict, name); - if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, - index == DKIX_EMPTY ? - SPEC_FAIL_ATTR_NOT_IN_DICT : - SPEC_FAIL_OUT_OF_RANGE); - return 0; - } - cache->index = (uint16_t)index; - write_u32(cache->version, type->tp_version_tag); - specialize(instr, hint_op); + int res; + Py_BEGIN_CRITICAL_SECTION(dict); + // materialized managed dict + res = specialize_dict_access_hint(dict, instr, type, kind, name, + tp_version, base_op, hint_op); + Py_END_CRITICAL_SECTION(); + return res; } - return 1; } #ifndef Py_GIL_DISABLED @@ -1050,7 +1106,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na PyTypeObject *type = Py_TYPE(owner); bool shadow = instance_has_key(owner, name); PyObject *descr = NULL; - DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0); + DescriptorClassification kind = analyze_descriptor_load(type, name, &descr); + Py_XDECREF(descr); // turn strong ref into a borrowed ref assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); if (type_get_version(type, LOAD_ATTR) == 0) { return -1; @@ -1204,8 +1261,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na } Py_UNREACHABLE(); try_instance: - if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR, - LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) + if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag, + LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) { return 0; } @@ -1259,8 +1316,9 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na { PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); + PyObject *descr = NULL; _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); if (!_PyType_IsReady(type)) { @@ -1274,11 +1332,12 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; } - PyObject *descr; - DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1); - if (type_get_version(type, STORE_ATTR) == 0) { + unsigned int tp_version = 0; + DescriptorClassification kind = analyze_descriptor_store(type, name, &descr, &tp_version); + if (tp_version == 0) { goto fail; } + assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1309,8 +1368,8 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); - instr->op.code = STORE_ATTR_SLOT; + write_u32(cache->version, tp_version); + specialize(instr, STORE_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -1337,22 +1396,19 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); goto fail; case ABSENT: - if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, - STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT)) - { + if (specialize_dict_access(owner, instr, type, kind, name, tp_version, + STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, + STORE_ATTR_WITH_HINT)) { goto success; } } fail: - STAT_INC(STORE_ATTR, failure); - assert(!PyErr_Occurred()); - instr->op.code = STORE_ATTR; - cache->counter = adaptive_counter_backoff(cache->counter); + Py_XDECREF(descr); + unspecialize(instr); return; success: - STAT_INC(STORE_ATTR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + Py_XDECREF(descr); + return; } #ifndef Py_GIL_DISABLED @@ -1421,7 +1477,8 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, } PyObject *descr = NULL; DescriptorClassification kind = 0; - kind = analyze_descriptor(cls, name, &descr, 0); + kind = analyze_descriptor_load(cls, name, &descr); + Py_XDECREF(descr); // turn strong ref into a borrowed ref if (type_get_version(cls, LOAD_ATTR) == 0) { return -1; } @@ -1714,7 +1771,6 @@ function_get_version(PyObject *o, int opcode) } return version; } -#endif // Py_GIL_DISABLED /* Returning 0 indicates a failure. */ static uint32_t @@ -1727,6 +1783,7 @@ type_get_version(PyTypeObject *t, int opcode) } return version; } +#endif // Py_GIL_DISABLED void _Py_Specialize_BinarySubscr( From e163e8d4e1a9844b8615ef38b9917b887a377948 Mon Sep 17 00:00:00 2001 From: Zhikang Yan <2951256653@qq.com> Date: Fri, 20 Dec 2024 04:24:47 +0800 Subject: [PATCH 249/427] gh-128062: Fix the font size and shortcut display of the turtledemo menu (#128063) Leave the font of the menu bar the default to keep it consistent with the rest of the world. Display the shortcut keys in the right way, using the 'accelerator' option. --------- Co-authored-by: Peter Bierma Co-authored-by: Terry Jan Reedy --- Lib/turtledemo/__main__.py | 17 +++++++---------- ...24-12-18-10-18-55.gh-issue-128062.E9oU7-.rst | 2 ++ 2 files changed, 9 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index 9c15916fb6672e..b49c0beab3ccf7 100644 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -105,7 +105,6 @@ DONE = 4 EVENTDRIVEN = 5 -menufont = ("Arial", 12, NORMAL) btnfont = ("Arial", 12, 'bold') txtfont = ['Lucida Console', 10, 'normal'] @@ -297,23 +296,21 @@ def makeLoadDemoMenu(self, master): for entry in getExampleEntries(): def load(entry=entry): self.loadfile(entry) - menu.add_command(label=entry, underline=0, - font=menufont, command=load) + menu.add_command(label=entry, underline=0, command=load) return menu def makeFontMenu(self, master): menu = Menu(master, tearoff=0) - menu.add_command(label="Decrease (C-'-')", command=self.decrease_size, - font=menufont) - menu.add_command(label="Increase (C-'+')", command=self.increase_size, - font=menufont) + menu.add_command(label="Decrease", command=self.decrease_size, + accelerator=f"{'Command' if darwin else 'Ctrl'}+-") + menu.add_command(label="Increase", command=self.increase_size, + accelerator=f"{'Command' if darwin else 'Ctrl'}+=") menu.add_separator() for size in font_sizes: def resize(size=size): self.set_txtsize(size) - menu.add_command(label=str(size), underline=0, - font=menufont, command=resize) + menu.add_command(label=str(size), underline=0, command=resize) return menu def makeHelpMenu(self, master): @@ -322,7 +319,7 @@ def makeHelpMenu(self, master): for help_label, help_file in help_entries: def show(help_label=help_label, help_file=help_file): view_text(self.root, help_label, help_file) - menu.add_command(label=help_label, font=menufont, command=show) + menu.add_command(label=help_label, command=show) return menu def refreshCanvas(self): diff --git a/Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst b/Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst new file mode 100644 index 00000000000000..d8e262e0848077 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-18-10-18-55.gh-issue-128062.E9oU7-.rst @@ -0,0 +1,2 @@ +Revert the font of :mod:`turtledemo`'s menu bar to its default value and +display the shortcut keys in the correct position. From 255762c09fe518757bb3e8ce1bb6e5d8eec9f466 Mon Sep 17 00:00:00 2001 From: mpage Date: Thu, 19 Dec 2024 13:03:14 -0800 Subject: [PATCH 250/427] gh-127274: Defer nested methods (#128012) Methods (functions defined in class scope) are likely to be cleaned up by the GC anyway. Add a new code flag, `CO_METHOD`, that is set for functions defined in a class scope. Use that when deciding to defer functions. --- Doc/library/inspect.rst | 7 +++++++ Include/cpython/code.h | 3 +++ Include/internal/pycore_symtable.h | 1 + Lib/dis.py | 1 + Lib/inspect.py | 1 + Lib/test/test_monitoring.py | 9 +++------ Lib/test/test_opcache.py | 11 ++++------- .../2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst | 3 +++ Objects/funcobject.c | 6 +++++- Python/compile.c | 2 ++ Python/symtable.c | 7 +++++++ 11 files changed, 37 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index ca5dac87aff2b4..0085207d3055f2 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1708,6 +1708,13 @@ which is a bitmap of the following flags: .. versionadded:: 3.14 +.. data:: CO_METHOD + + The flag is set when the code object is a function defined in class + scope. + + .. versionadded:: 3.14 + .. note:: The flags are specific to CPython, and may not be defined in other Python implementations. Furthermore, the flags are an implementation diff --git a/Include/cpython/code.h b/Include/cpython/code.h index c3c0165d556ead..cb6261ddde941b 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -199,6 +199,9 @@ struct PyCodeObject _PyCode_DEF(1); */ #define CO_HAS_DOCSTRING 0x4000000 +/* A function defined in class scope */ +#define CO_METHOD 0x8000000 + /* This should be defined if a future statement modifies the syntax. For example, when a keyword is added. */ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 91dac767d5885b..b7e274296112aa 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -124,6 +124,7 @@ typedef struct _symtable_entry { unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an enclosing class scope */ unsigned ste_has_docstring : 1; /* true if docstring present */ + unsigned ste_method : 1; /* true if block is a function block defined in class scope */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ _Py_SourceLocation ste_loc; /* source location of block */ struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ diff --git a/Lib/dis.py b/Lib/dis.py index aa22404c6687e1..109c986bbe3d7d 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -162,6 +162,7 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets 256: "ITERABLE_COROUTINE", 512: "ASYNC_GENERATOR", 0x4000000: "HAS_DOCSTRING", + 0x8000000: "METHOD", } def pretty_flags(flags): diff --git a/Lib/inspect.py b/Lib/inspect.py index b7d8271f8a471f..5b7c4df8927c87 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -57,6 +57,7 @@ "CO_VARARGS", "CO_VARKEYWORDS", "CO_HAS_DOCSTRING", + "CO_METHOD", "ClassFoundException", "ClosureVars", "EndOfBlock", diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 087ac8d456b843..32b3a6ac049e28 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -850,12 +850,6 @@ def __init__(self, events): def __call__(self, code, offset, val): self.events.append(("return", code.co_name, val)) -# gh-127274: CALL_ALLOC_AND_ENTER_INIT will only cache __init__ methods that -# are deferred. We only defer functions defined at the top-level. -class ValueErrorRaiser: - def __init__(self): - raise ValueError() - class ExceptionMonitoringTest(CheckEvents): @@ -1054,6 +1048,9 @@ def func(): @requires_specialization_ft def test_no_unwind_for_shim_frame(self): + class ValueErrorRaiser: + def __init__(self): + raise ValueError() def f(): try: diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index ad0b0c487a4118..ba111b5117b41d 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -493,13 +493,6 @@ def f(): self.assertFalse(f()) -# gh-127274: CALL_ALLOC_AND_ENTER_INIT will only cache __init__ methods that -# are deferred. We only defer functions defined at the top-level. -class MyClass: - def __init__(self): - pass - - class InitTakesArg: def __init__(self, arg): self.arg = arg @@ -536,6 +529,10 @@ def f(x, y): @disabling_optimizer @requires_specialization_ft def test_assign_init_code(self): + class MyClass: + def __init__(self): + pass + def instantiate(): return MyClass() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst new file mode 100644 index 00000000000000..a4608fbbbf19ec --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-13-45-33.gh-issue-127274.deNxNC.rst @@ -0,0 +1,3 @@ +Add a new flag, ``CO_METHOD``, to :attr:`~codeobject.co_flags` that +indicates whether the code object belongs to a function defined in class +scope. diff --git a/Objects/funcobject.c b/Objects/funcobject.c index cca7f01498013e..7b17a9ba31fac4 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -210,10 +210,14 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = FUNC_VERSION_UNSET; - if ((code_obj->co_flags & CO_NESTED) == 0) { + if (((code_obj->co_flags & CO_NESTED) == 0) || + (code_obj->co_flags & CO_METHOD)) { // Use deferred reference counting for top-level functions, but not // nested functions because they are more likely to capture variables, // which makes prompt deallocation more important. + // + // Nested methods (functions defined in class scope) are also deferred, + // since they will likely be cleaned up by GC anyway. _PyObject_SetDeferredRefcount((PyObject *)op); } _PyObject_GC_TRACK(op); diff --git a/Python/compile.c b/Python/compile.c index cbfba7f493e07d..ef470830336dde 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1289,6 +1289,8 @@ compute_code_flags(compiler *c) flags |= CO_VARKEYWORDS; if (ste->ste_has_docstring) flags |= CO_HAS_DOCSTRING; + if (ste->ste_method) + flags |= CO_METHOD; } if (ste->ste_coroutine && !ste->ste_generator) { diff --git a/Python/symtable.c b/Python/symtable.c index ebddb0b93fca0a..49bd01ba68ac9e 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -138,6 +138,13 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_has_docstring = 0; + ste->ste_method = 0; + if (st->st_cur != NULL && + st->st_cur->ste_type == ClassBlock && + block == FunctionBlock) { + ste->ste_method = 1; + } + ste->ste_symbols = PyDict_New(); ste->ste_varnames = PyList_New(0); ste->ste_children = PyList_New(0); From c14db202750ff9eaf3919298f1172270b7dfd64e Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Thu, 19 Dec 2024 14:07:17 -0800 Subject: [PATCH 251/427] gh-128080: remove unnecessary `__init__` method from Enum (GH-128081) remove unnecessary __init__ method from Enum Co-authored-by: Peter Bierma --- Lib/enum.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/enum.py b/Lib/enum.py index 27be3fb83b2afb..ccc1da42206474 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1211,9 +1211,6 @@ def __new__(cls, value): exc = None ve_exc = None - def __init__(self, *args, **kwds): - pass - def _add_alias_(self, name): self.__class__._add_member_(name, self) From 39e69a7cd54d44c9061db89bb15c460d30fba7a6 Mon Sep 17 00:00:00 2001 From: Md Rokibul Islam Date: Fri, 20 Dec 2024 00:38:42 +0100 Subject: [PATCH 252/427] gh-112328: Document EnumDict in docs and release notes (GH-121720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) Co-authored-by: Ethan Furman --- Doc/library/enum.rst | 17 ++++++++++++++++- Doc/whatsnew/3.13.rst | 5 +++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 16a9b0326e9f3d..51292a11f507c4 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -149,9 +149,14 @@ Module Contents Return a list of all power-of-two integers contained in a flag. + :class:`EnumDict` + + A subclass of :class:`dict` for use when subclassing :class:`EnumType`. + .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` .. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` +.. versionadded:: 3.14 ``EnumDict`` --------------- @@ -821,7 +826,17 @@ Data Types >>> KeepFlag(2**2 + 2**4) -.. versionadded:: 3.11 + .. versionadded:: 3.11 + +.. class:: EnumDict + + *EnumDict* is a subclass of :class:`dict` for use when subclassing :class:`EnumType`. + + .. attribute:: EnumDict.member_names + + Return list of member names. + + .. versionadded:: 3.14 --------------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 45cc1b5bad9b18..a291122aefc2ce 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -879,6 +879,11 @@ email (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve the :cve:`2023-27043` fix.) +enum +---- + +* :class:`~enum.EnumDict` has been made public in :mod:`enum` to better support + subclassing :class:`~enum.EnumType`. fractions --------- From daa260ebb1c1b20321e7f26df7c9dbd35d4edcbf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Dec 2024 08:50:18 +0100 Subject: [PATCH 253/427] gh-128058: Fix test_builtin ImmortalTests (#128068) On 32-bit Free Threading systems, immortal reference count is 5 << 28, instead of 7 << 28. Co-authored-by: Peter Bierma --- Lib/test/test_builtin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index a92edad86839e6..f98138391bc1a8 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2691,7 +2691,10 @@ def __del__(self): class ImmortalTests(unittest.TestCase): if sys.maxsize < (1 << 32): - IMMORTAL_REFCOUNT = 7 << 28 + if support.Py_GIL_DISABLED: + IMMORTAL_REFCOUNT = 5 << 28 + else: + IMMORTAL_REFCOUNT = 7 << 28 else: IMMORTAL_REFCOUNT = 3 << 30 From 45e6dd63b88a782f2ec96ab1da54eb5a074d8f4c Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 20 Dec 2024 00:22:26 -0800 Subject: [PATCH 254/427] gh-128030: Avoid error from PyModule_GetFilenameObject for non-module (#128047) I missed the extra `PyModule_Check` in #127660 because I was looking at 3.12 as the base implementation for import from. This meant that I missed the `PyModuleCheck` introduced in #112661. --- Lib/test/test_import/__init__.py | 23 +++++++++++++++++++ ...-12-17-22-28-15.gh-issue-128030.H1ptOD.rst | 1 + Python/ceval.c | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 83efbc1e25e77a..c2cec6444cb43a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -851,6 +851,29 @@ def test_frozen_module_from_import_error(self): stdout, stderr = popen.communicate() self.assertIn(expected_error, stdout) + def test_non_module_from_import_error(self): + prefix = """ +import sys +class NotAModule: ... +nm = NotAModule() +nm.symbol = 123 +sys.modules["not_a_module"] = nm +from not_a_module import symbol +""" + scripts = [ + prefix + "from not_a_module import missing_symbol", + prefix + "nm.__spec__ = []\nfrom not_a_module import missing_symbol", + ] + for script in scripts: + with self.subTest(script=script): + expected_error = ( + b"ImportError: cannot import name 'missing_symbol' from " + b"'' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + def test_script_shadowing_stdlib(self): script_errors = [ ( diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst new file mode 100644 index 00000000000000..93d78632355b76 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-22-28-15.gh-issue-128030.H1ptOD.rst @@ -0,0 +1 @@ +Avoid error from calling ``PyModule_GetFilenameObject`` on a non-module object when importing a non-existent symbol from a non-module object. diff --git a/Python/ceval.c b/Python/ceval.c index fd891d7839151e..bfdf5687c287db 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2860,7 +2860,7 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) } } - if (origin == NULL) { + if (origin == NULL && PyModule_Check(v)) { // Fall back to __file__ for diagnostics if we don't have // an origin that is a location origin = PyModule_GetFilenameObject(v); From df46c780febab667ee01264ae32c4e866cecd911 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Dec 2024 11:57:44 +0000 Subject: [PATCH 255/427] GH-122548: Correct magic number comment (GH-128115) Correct magic number comment --- Include/internal/pycore_magic_number.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 079a9befcd4c5e..4c3b9c4c71da1b 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -262,7 +262,8 @@ Known values: Python 3.14a1 3607 (Add pseudo instructions JUMP_IF_TRUE/FALSE) Python 3.14a1 3608 (Add support for slices) Python 3.14a2 3609 (Add LOAD_SMALL_INT and LOAD_CONST_IMMORTAL instructions, remove RETURN_CONST) - Python 3.14a3 3610 (Add NOT_TAKEN instruction) + (3610 accidentally omitted) + Python 3.14a4 3611 (Add NOT_TAKEN instruction) Python 3.15 will start with 3650 From 0974d7bb866062ed4aaa40f705d6cc4c294d99f1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Dec 2024 13:37:20 +0100 Subject: [PATCH 256/427] gh-109959: Log the current directory in test_glob.test_selflink() (#128122) --- Lib/test/test_glob.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index b72640bd871ba6..00187a3fb3537d 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -6,6 +6,7 @@ import unittest import warnings +from test import support from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) @@ -515,6 +516,12 @@ def test_selflink(self): os.makedirs(tempdir) self.addCleanup(shutil.rmtree, tempdir) with change_cwd(tempdir): + if support.verbose: + cwd = os.getcwd() + print(f"cwd: {cwd} ({len(cwd)} chars)") + cwdb = os.getcwdb() + print(f"cwdb: {cwdb!r} ({len(cwdb)} bytes)") + os.makedirs('dir') create_empty_file(os.path.join('dir', 'file')) os.symlink(os.curdir, os.path.join('dir', 'link')) From cbfe3023e46b544b80ea1a38a8c900c6fb881554 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Dec 2024 13:38:00 +0100 Subject: [PATCH 257/427] gh-128116: Skip test_socket VSOCK testStream() on PermissionError (#128120) --- Lib/test/test_socket.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 307d6e886c617f..aac213e36aecf0 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -547,7 +547,10 @@ def clientSetUp(self): self.cli.connect((cid, VSOCKPORT)) def testStream(self): - msg = self.conn.recv(1024) + try: + msg = self.conn.recv(1024) + except PermissionError as exc: + self.skipTest(repr(exc)) self.assertEqual(msg, MSG) def _testStream(self): From ba45e5cdd41a39ce0b3de08bdcfa9d8e28e0e4f3 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 20 Dec 2024 08:02:46 -0500 Subject: [PATCH 258/427] gh-127946: Use a critical section for `CFuncPtr` attributes (GH-128109) --- Lib/test/test_ctypes/test_cfuncs.py | 20 +- ...-12-19-20-46-01.gh-issue-127946.4lM3Op.rst | 2 + Modules/_ctypes/_ctypes.c | 100 +++++++--- Modules/_ctypes/clinic/_ctypes.c.h | 174 +++++++++++++++++- 4 files changed, 266 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst diff --git a/Lib/test/test_ctypes/test_cfuncs.py b/Lib/test/test_ctypes/test_cfuncs.py index 48330c4b0a763b..e0c124607cb2e9 100644 --- a/Lib/test/test_ctypes/test_cfuncs.py +++ b/Lib/test/test_ctypes/test_cfuncs.py @@ -5,7 +5,8 @@ c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) -from test.support import import_helper +from test import support +from test.support import import_helper, threading_helper _ctypes_test = import_helper.import_module("_ctypes_test") @@ -191,6 +192,23 @@ def test_void(self): self.assertEqual(self._dll.tv_i(-42), None) self.assertEqual(self.S(), -42) + @threading_helper.requires_working_threading() + @support.requires_resource("cpu") + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful on free-threading") + def test_thread_safety(self): + from threading import Thread + + def concurrent(): + for _ in range(100): + self._dll.tf_b.restype = c_byte + self._dll.tf_b.argtypes = (c_byte,) + + with threading_helper.catch_threading_exception() as exc: + with threading_helper.start_threads((Thread(target=concurrent) for _ in range(10))): + pass + + self.assertIsNone(exc.exc_value) + # The following repeats the above tests with stdcall functions (where # they are available) diff --git a/Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst b/Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst new file mode 100644 index 00000000000000..faf1ec042bc2b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-19-20-46-01.gh-issue-127946.4lM3Op.rst @@ -0,0 +1,2 @@ +Fix crash when modifying :class:`ctypes._CFuncPtr` objects concurrently on +the :term:`free threaded ` build. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 3a3b1da5084a67..dcdb7b2052a11e 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -128,8 +128,9 @@ bytes(cdata) /*[clinic input] module _ctypes +class _ctypes.CFuncPtr "PyCFuncPtrObject *" "&PyCFuncPtr_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=58e8c99474bc631e]*/ #define clinic_state() (get_module_state_by_class(cls)) #define clinic_state_sub() (get_module_state_by_class(cls->tp_base)) @@ -3422,21 +3423,37 @@ generic_pycdata_new(ctypes_state *st, PyCFuncPtr_Type */ +/*[clinic input] +@critical_section +@setter +_ctypes.CFuncPtr.errcheck +[clinic start generated code]*/ + static int -PyCFuncPtr_set_errcheck(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_errcheck_set_impl(PyCFuncPtrObject *self, PyObject *value) +/*[clinic end generated code: output=6580cf1ffdf3b9fb input=84930bb16c490b33]*/ { - if (ob && !PyCallable_Check(ob)) { + if (value && !PyCallable_Check(value)) { PyErr_SetString(PyExc_TypeError, "the errcheck attribute must be callable"); return -1; } - Py_XINCREF(ob); - Py_XSETREF(self->errcheck, ob); + Py_XINCREF(value); + Py_XSETREF(self->errcheck, value); return 0; } +/*[clinic input] +@critical_section +@getter +_ctypes.CFuncPtr.errcheck + +a function to check for errors +[clinic start generated code]*/ + static PyObject * -PyCFuncPtr_get_errcheck(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_errcheck_get_impl(PyCFuncPtrObject *self) +/*[clinic end generated code: output=dfa6fb5c6f90fd14 input=4672135fef37819f]*/ { if (self->errcheck) { return Py_NewRef(self->errcheck); @@ -3444,11 +3461,18 @@ PyCFuncPtr_get_errcheck(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) Py_RETURN_NONE; } +/*[clinic input] +@setter +@critical_section +_ctypes.CFuncPtr.restype +[clinic start generated code]*/ + static int -PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value) +/*[clinic end generated code: output=0be0a086abbabf18 input=683c3bef4562ccc6]*/ { PyObject *checker, *oldchecker; - if (ob == NULL) { + if (value == NULL) { oldchecker = self->checker; self->checker = NULL; Py_CLEAR(self->restype); @@ -3457,27 +3481,36 @@ PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ign } ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *info; - if (PyStgInfo_FromType(st, ob, &info) < 0) { + if (PyStgInfo_FromType(st, value, &info) < 0) { return -1; } - if (ob != Py_None && !info && !PyCallable_Check(ob)) { + if (value != Py_None && !info && !PyCallable_Check(value)) { PyErr_SetString(PyExc_TypeError, "restype must be a type, a callable, or None"); return -1; } - if (PyObject_GetOptionalAttr(ob, &_Py_ID(_check_retval_), &checker) < 0) { + if (PyObject_GetOptionalAttr(value, &_Py_ID(_check_retval_), &checker) < 0) { return -1; } oldchecker = self->checker; self->checker = checker; - Py_INCREF(ob); - Py_XSETREF(self->restype, ob); + Py_INCREF(value); + Py_XSETREF(self->restype, value); Py_XDECREF(oldchecker); return 0; } +/*[clinic input] +@getter +@critical_section +_ctypes.CFuncPtr.restype + +specify the result type +[clinic start generated code]*/ + static PyObject * -PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_restype_get_impl(PyCFuncPtrObject *self) +/*[clinic end generated code: output=c8f44cd16f1dee5e input=5e3ed95116204fd2]*/ { if (self->restype) { return Py_NewRef(self->restype); @@ -3495,28 +3528,44 @@ PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@setter +@critical_section +_ctypes.CFuncPtr.argtypes +[clinic start generated code]*/ + static int -PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_argtypes_set_impl(PyCFuncPtrObject *self, PyObject *value) +/*[clinic end generated code: output=596a36e2ae89d7d1 input=c4627573e980aa8b]*/ { PyObject *converters; - if (ob == NULL || ob == Py_None) { + if (value == NULL || value == Py_None) { Py_CLEAR(self->converters); Py_CLEAR(self->argtypes); } else { ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); - converters = converters_from_argtypes(st, ob); + converters = converters_from_argtypes(st, value); if (!converters) return -1; Py_XSETREF(self->converters, converters); - Py_INCREF(ob); - Py_XSETREF(self->argtypes, ob); + Py_INCREF(value); + Py_XSETREF(self->argtypes, value); } return 0; } +/*[clinic input] +@getter +@critical_section +_ctypes.CFuncPtr.argtypes + +specify the argument types +[clinic start generated code]*/ + static PyObject * -PyCFuncPtr_get_argtypes(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_argtypes_get_impl(PyCFuncPtrObject *self) +/*[clinic end generated code: output=c46b05a1b0f99172 input=37a8a545a56f8ae2]*/ { if (self->argtypes) { return Py_NewRef(self->argtypes); @@ -3535,13 +3584,9 @@ PyCFuncPtr_get_argtypes(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) } static PyGetSetDef PyCFuncPtr_getsets[] = { - { "errcheck", (getter)PyCFuncPtr_get_errcheck, (setter)PyCFuncPtr_set_errcheck, - "a function to check for errors", NULL }, - { "restype", (getter)PyCFuncPtr_get_restype, (setter)PyCFuncPtr_set_restype, - "specify the result type", NULL }, - { "argtypes", (getter)PyCFuncPtr_get_argtypes, - (setter)PyCFuncPtr_set_argtypes, - "specify the argument types", NULL }, + _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF + _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF + _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF { NULL, NULL } }; @@ -5054,7 +5099,6 @@ class _ctypes.Simple "PyObject *" "clinic_state()->Simple_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=016c476c7aa8b8a8]*/ - static int Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) { diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h index 1332ba04cdfecd..405a3c9238d77d 100644 --- a/Modules/_ctypes/clinic/_ctypes.c.h +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_ctypes_CType_Type___sizeof____doc__, @@ -601,6 +602,177 @@ PyCData_reduce(PyObject *myself, PyTypeObject *cls, PyObject *const *args, Py_ss return PyCData_reduce_impl(myself, cls); } +#if !defined(_ctypes_CFuncPtr_errcheck_DOCSTR) +# define _ctypes_CFuncPtr_errcheck_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", (getter)_ctypes_CFuncPtr_errcheck_get, (setter)_ctypes_CFuncPtr_errcheck_set, _ctypes_CFuncPtr_errcheck_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", NULL, (setter)_ctypes_CFuncPtr_errcheck_set, NULL}, +#endif + +static int +_ctypes_CFuncPtr_errcheck_set_impl(PyCFuncPtrObject *self, PyObject *value); + +static int +_ctypes_CFuncPtr_errcheck_set(PyCFuncPtrObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_errcheck_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ctypes_CFuncPtr_errcheck__doc__, +"a function to check for errors"); +#if defined(_ctypes_CFuncPtr_errcheck_DOCSTR) +# undef _ctypes_CFuncPtr_errcheck_DOCSTR +#endif +#define _ctypes_CFuncPtr_errcheck_DOCSTR _ctypes_CFuncPtr_errcheck__doc__ + +#if !defined(_ctypes_CFuncPtr_errcheck_DOCSTR) +# define _ctypes_CFuncPtr_errcheck_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", (getter)_ctypes_CFuncPtr_errcheck_get, (setter)_ctypes_CFuncPtr_errcheck_set, _ctypes_CFuncPtr_errcheck_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", (getter)_ctypes_CFuncPtr_errcheck_get, NULL, _ctypes_CFuncPtr_errcheck_DOCSTR}, +#endif + +static PyObject * +_ctypes_CFuncPtr_errcheck_get_impl(PyCFuncPtrObject *self); + +static PyObject * +_ctypes_CFuncPtr_errcheck_get(PyCFuncPtrObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_errcheck_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ctypes_CFuncPtr_restype_DOCSTR) +# define _ctypes_CFuncPtr_restype_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_RESTYPE_GETSETDEF) +# undef _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", (getter)_ctypes_CFuncPtr_restype_get, (setter)_ctypes_CFuncPtr_restype_set, _ctypes_CFuncPtr_restype_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", NULL, (setter)_ctypes_CFuncPtr_restype_set, NULL}, +#endif + +static int +_ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value); + +static int +_ctypes_CFuncPtr_restype_set(PyCFuncPtrObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_restype_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ctypes_CFuncPtr_restype__doc__, +"specify the result type"); +#if defined(_ctypes_CFuncPtr_restype_DOCSTR) +# undef _ctypes_CFuncPtr_restype_DOCSTR +#endif +#define _ctypes_CFuncPtr_restype_DOCSTR _ctypes_CFuncPtr_restype__doc__ + +#if !defined(_ctypes_CFuncPtr_restype_DOCSTR) +# define _ctypes_CFuncPtr_restype_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_RESTYPE_GETSETDEF) +# undef _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", (getter)_ctypes_CFuncPtr_restype_get, (setter)_ctypes_CFuncPtr_restype_set, _ctypes_CFuncPtr_restype_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", (getter)_ctypes_CFuncPtr_restype_get, NULL, _ctypes_CFuncPtr_restype_DOCSTR}, +#endif + +static PyObject * +_ctypes_CFuncPtr_restype_get_impl(PyCFuncPtrObject *self); + +static PyObject * +_ctypes_CFuncPtr_restype_get(PyCFuncPtrObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_restype_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ctypes_CFuncPtr_argtypes_DOCSTR) +# define _ctypes_CFuncPtr_argtypes_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", (getter)_ctypes_CFuncPtr_argtypes_get, (setter)_ctypes_CFuncPtr_argtypes_set, _ctypes_CFuncPtr_argtypes_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", NULL, (setter)_ctypes_CFuncPtr_argtypes_set, NULL}, +#endif + +static int +_ctypes_CFuncPtr_argtypes_set_impl(PyCFuncPtrObject *self, PyObject *value); + +static int +_ctypes_CFuncPtr_argtypes_set(PyCFuncPtrObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_argtypes_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ctypes_CFuncPtr_argtypes__doc__, +"specify the argument types"); +#if defined(_ctypes_CFuncPtr_argtypes_DOCSTR) +# undef _ctypes_CFuncPtr_argtypes_DOCSTR +#endif +#define _ctypes_CFuncPtr_argtypes_DOCSTR _ctypes_CFuncPtr_argtypes__doc__ + +#if !defined(_ctypes_CFuncPtr_argtypes_DOCSTR) +# define _ctypes_CFuncPtr_argtypes_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", (getter)_ctypes_CFuncPtr_argtypes_get, (setter)_ctypes_CFuncPtr_argtypes_set, _ctypes_CFuncPtr_argtypes_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", (getter)_ctypes_CFuncPtr_argtypes_get, NULL, _ctypes_CFuncPtr_argtypes_DOCSTR}, +#endif + +static PyObject * +_ctypes_CFuncPtr_argtypes_get_impl(PyCFuncPtrObject *self); + +static PyObject * +_ctypes_CFuncPtr_argtypes_get(PyCFuncPtrObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_argtypes_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(Simple_from_outparm__doc__, "__ctypes_from_outparam__($self, /)\n" "--\n" @@ -621,4 +793,4 @@ Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py } return Simple_from_outparm_impl(self, cls); } -/*[clinic end generated code: output=52724c091e3a8b8d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cb3583522a2c5ce5 input=a9049054013a1b77]*/ From 78ffba4221dcb2e39fd5db80c297d1777588bb59 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 20 Dec 2024 14:28:18 +0100 Subject: [PATCH 259/427] gh-127295: ctypes: Switch field accessors to fixed-width integers (GH-127297) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should be a pure refactoring, without user-visible behaviour changes. Before this change, ctypes uses traditional native C types, usually identified by [`struct` format characters][struct-chars] when a short (and identifier-friendly) name is needed: - `signed char` (`b`) / `unsigned char` (`B`) - `short` (`h`) / `unsigned short` (`h`) - `int` (`i`) / `unsigned int` (`i`) - `long` (`l`) / `unsigned long` (`l`) - `long long` (`q`) / `unsigned long long` (`q`) These map to C99 fixed-width types, which this PR switches to: - - `int8_t`/`uint8_t` - `int16_t`/`uint16_t` - `int32_t`/`uint32_t` - `int64_t`/`uint64_t` The C standard doesn't guarantee that the “traditional” types must map to the fixints. But, [`ctypes` currently requires it][swapdefs], so the assumption won't break anything. By “map” I mean that the *size* of the types matches. The *alignment* requirements might not. This needs to be kept in mind but is not an issue in `ctypes` accessors, which [explicitly handle unaligned memory][memcpy] for the integer types. Note that there are 5 “traditional” C type sizes, but 4 fixed-width ones. Two of the former are functionally identical to one another; which ones they are is platform-specific (e.g. `int`==`long`==`int32_t`.) This means that one of the [current][current-impls-1] [implementations][current-impls-2] is redundant on any given platform. The fixint types are parametrized by the number of bytes/bits, and one bit for signedness. This makes it easier to autogenerate code for them or to write generic macros (though generic API like [`PyLong_AsNativeBytes`][PyLong_AsNativeBytes] is problematic for performance reasons -- especially compared to a `memcpy` with compile-time-constant size). When one has a *different* integer type, determining the corresponding fixint means a `sizeof` and signedness check. This is easier and more robust than the current implementations (see [`wchar_t`][sizeof-wchar_t] or [`_Bool`][sizeof-bool]). [swapdefs]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L420-L444 [struct-chars]: https://docs.python.org/3/library/struct.html#format-characters [current-impls-1]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L470-L653 [current-impls-2]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L703-L944 [memcpy]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L613 [PyLong_AsNativeBytes]: https://docs.python.org/3/c-api/long.html#c.PyLong_AsNativeBytes [sizeof-wchar_t]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L1547-L1555 [sizeof-bool]: https://github.com/python/cpython/blob/v3.13.0/Modules/_ctypes/cfield.c#L1562-L1572 Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/_ctypes/_ctypes.c | 4 +- Modules/_ctypes/callbacks.c | 2 +- Modules/_ctypes/cfield.c | 1132 +++++++++++++++-------------------- Modules/_ctypes/ctypes.h | 15 +- 4 files changed, 501 insertions(+), 652 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index dcdb7b2052a11e..ac520ffaad6c90 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1979,7 +1979,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) return NULL; parg->pffi_type = &ffi_type_pointer; parg->tag = 'P'; - parg->obj = fd->setfunc(&parg->value, value, 0); + parg->obj = fd->setfunc(&parg->value, value, sizeof(void*)); if (parg->obj == NULL) { Py_DECREF(parg); return NULL; @@ -2444,7 +2444,7 @@ PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, parg->tag = fmt[0]; parg->pffi_type = fd->pffi_type; - parg->obj = fd->setfunc(&parg->value, value, 0); + parg->obj = fd->setfunc(&parg->value, value, info->size); if (parg->obj) return (PyObject *)parg; PyObject *exc = PyErr_GetRaisedException(); diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 7b9f6437c7d55f..89c0749a093765 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -264,7 +264,7 @@ static void _CallPythonObject(ctypes_state *st, be the result. EXCEPT when restype is py_object - Python itself knows how to manage the refcount of these objects. */ - PyObject *keep = setfunc(mem, result, 0); + PyObject *keep = setfunc(mem, result, restype->size); if (keep == NULL) { /* Could not convert callback result. */ diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 2b9e8a1a10d6f5..dcac9da75360a4 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -10,6 +10,7 @@ #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include // bool #include #include "ctypes.h" @@ -320,61 +321,6 @@ PyType_Spec cfield_spec = { }; -/******************************************************************/ -/* - Accessor functions -*/ - -/* Derived from Modules/structmodule.c: - Helper routine to get a Python integer and raise the appropriate error - if it isn't one */ - -static int -get_long(PyObject *v, long *p) -{ - long x = PyLong_AsUnsignedLongMask(v); - if (x == -1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - -/* Same, but handling unsigned long */ - -static int -get_ulong(PyObject *v, unsigned long *p) -{ - unsigned long x = PyLong_AsUnsignedLongMask(v); - if (x == (unsigned long)-1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - -/* Same, but handling native long long. */ - -static int -get_longlong(PyObject *v, long long *p) -{ - long long x = PyLong_AsUnsignedLongLongMask(v); - if (x == -1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - -/* Same, but handling native unsigned long long. */ - -static int -get_ulonglong(PyObject *v, unsigned long long *p) -{ - unsigned long long x = PyLong_AsUnsignedLongLongMask(v); - if (x == (unsigned long long)-1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - /***************************************************************** * Integer fields, with bitfield support */ @@ -404,34 +350,8 @@ Py_ssize_t NUM_BITS(Py_ssize_t bitsize) { /* This macro RETURNS the first parameter with the bit field CHANGED. */ #define SET(type, x, v, size) \ (NUM_BITS(size) ? \ - ( ( (type)x & ~(BIT_MASK(type, size) << LOW_BIT(size)) ) | ( ((type)v & BIT_MASK(type, size)) << LOW_BIT(size) ) ) \ - : (type)v) - -#if SIZEOF_SHORT == 2 -# define SWAP_SHORT _Py_bswap16 -#else -# error "unsupported short size" -#endif - -#if SIZEOF_INT == 4 -# define SWAP_INT _Py_bswap32 -#else -# error "unsupported int size" -#endif - -#if SIZEOF_LONG == 4 -# define SWAP_LONG _Py_bswap32 -#elif SIZEOF_LONG == 8 -# define SWAP_LONG _Py_bswap64 -#else -# error "unsupported long size" -#endif - -#if SIZEOF_LONG_LONG == 8 -# define SWAP_LONG_LONG _Py_bswap64 -#else -# error "unsupported long long size" -#endif + ( ( (type)(x) & ~(BIT_MASK(type, size) << LOW_BIT(size)) ) | ( ((type)(v) & BIT_MASK(type, size)) << LOW_BIT(size) ) ) \ + : (type)(v)) /***************************************************************** * The setter methods return an object which must be kept alive, to keep the @@ -454,203 +374,145 @@ Py_ssize_t NUM_BITS(Py_ssize_t bitsize) { #endif /***************************************************************** - * integer accessor methods, supporting bit fields + * accessor methods for fixed-width integers (e.g. int8_t, uint64_t), + * supporting bit fields. + * These are named e.g. `i8_set`/`i8_get` or `u64_set`/`u64_get`, + * and are all alike, so they're defined using a macro. */ -static PyObject * -b_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - if (get_long(value, &val) < 0) - return NULL; - *(signed char *)ptr = SET(signed char, *(signed char *)ptr, val, size); - _RET(value); -} - - -static PyObject * -b_get(void *ptr, Py_ssize_t size) -{ - signed char val = *(signed char *)ptr; - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -B_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - if (get_ulong(value, &val) < 0) - return NULL; - *(unsigned char *)ptr = SET(unsigned char, *(unsigned char*)ptr, val, size); - _RET(value); -} - - -static PyObject * -B_get(void *ptr, Py_ssize_t size) -{ - unsigned char val = *(unsigned char *)ptr; - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -h_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - short x; - if (get_long(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(short, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - - -static PyObject * -h_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - short field; - if (get_long(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_SHORT(field); - field = SET(short, field, val, size); - field = SWAP_SHORT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - -static PyObject * -h_get(void *ptr, Py_ssize_t size) -{ - short val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong((long)val); -} - -static PyObject * -h_get_sw(void *ptr, Py_ssize_t size) -{ - short val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_SHORT(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -H_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned short x; - if (get_ulong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(unsigned short, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -H_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned short field; - if (get_ulong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_SHORT(field); - field = SET(unsigned short, field, val, size); - field = SWAP_SHORT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - - -static PyObject * -H_get(void *ptr, Py_ssize_t size) -{ - unsigned short val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -H_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned short val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_SHORT(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -i_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - int x; - if (get_long(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(int, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -i_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - int field; - if (get_long(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_INT(field); - field = SET(int, field, val, size); - field = SWAP_INT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} +#define FIXINT_GETSET(TAG, CTYPE, NBITS, PYAPI_FROMFUNC) \ + static PyObject * \ + TAG ## _set(void *ptr, PyObject *value, Py_ssize_t size_arg) \ + { \ + assert(NUM_BITS(size_arg) || (size_arg == (NBITS) / 8)); \ + CTYPE val; \ + if (PyLong_Check(value) \ + && PyUnstable_Long_IsCompact((PyLongObject *)value)) \ + { \ + val = (CTYPE)PyUnstable_Long_CompactValue( \ + (PyLongObject *)value); \ + } \ + else { \ + Py_ssize_t res = PyLong_AsNativeBytes( \ + value, &val, (NBITS) / 8, \ + Py_ASNATIVEBYTES_NATIVE_ENDIAN \ + | Py_ASNATIVEBYTES_ALLOW_INDEX); \ + if (res < 0) { \ + return NULL; \ + } \ + } \ + CTYPE prev; \ + memcpy(&prev, ptr, (NBITS) / 8); \ + val = SET(CTYPE, prev, val, size_arg); \ + memcpy(ptr, &val, (NBITS) / 8); \ + _RET(value); \ + } \ + \ + static PyObject * \ + TAG ## _get(void *ptr, Py_ssize_t size_arg) \ + { \ + assert(NUM_BITS(size_arg) || (size_arg == (NBITS) / 8)); \ + CTYPE val; \ + memcpy(&val, ptr, sizeof(val)); \ + GET_BITFIELD(val, size_arg); \ + return PYAPI_FROMFUNC(val); \ + } \ + /////////////////////////////////////////////////////////////////////////// + +/* Another macro for byte-swapped variants (e.g. `i8_set_sw`/`i8_get_sw`) */ + +#define FIXINT_GETSET_SW(TAG, CTYPE, NBITS, PYAPI_FROMFUNC, PY_SWAPFUNC) \ + static PyObject * \ + TAG ## _set_sw(void *ptr, PyObject *value, Py_ssize_t size_arg) \ + { \ + CTYPE val; \ + PyObject *res = TAG ## _set(&val, value, (NBITS) / 8); \ + if (res == NULL) { \ + return NULL; \ + } \ + Py_DECREF(res); \ + CTYPE field; \ + memcpy(&field, ptr, sizeof(field)); \ + field = PY_SWAPFUNC(field); \ + field = SET(CTYPE, field, val, size_arg); \ + field = PY_SWAPFUNC(field); \ + memcpy(ptr, &field, sizeof(field)); \ + _RET(value); \ + } \ + \ + static PyObject * \ + TAG ## _get_sw(void *ptr, Py_ssize_t size_arg) \ + { \ + assert(NUM_BITS(size_arg) || (size_arg == (NBITS) / 8)); \ + CTYPE val; \ + memcpy(&val, ptr, sizeof(val)); \ + val = PY_SWAPFUNC(val); \ + GET_BITFIELD(val, size_arg); \ + return PYAPI_FROMFUNC(val); \ + } \ + /////////////////////////////////////////////////////////////////////////// + +/* These macros are expanded for all supported combinations of byte sizes + * (1, 2, 4, 8), signed and unsigned, native and swapped byteorder. + * That's a lot, so generate the list with Argument Clinic (`make clinic`). + */ +/*[python input] +for nbits in 8, 16, 32, 64: + for sgn in 'i', 'u': + u = 'u' if sgn == 'u' else '' + U = u.upper() + apibits = max(nbits, 32) + parts = [ + f'{sgn}{nbits}', + f'{u}int{nbits}_t', + f'{nbits}', + f'PyLong_From{U}Int{apibits}', + ] + print(f'FIXINT_GETSET({", ".join(parts)})') + if nbits > 8: + parts.append(f'_Py_bswap{nbits}') + print(f'FIXINT_GETSET_SW({", ".join(parts)})') +[python start generated code]*/ +FIXINT_GETSET(i8, int8_t, 8, PyLong_FromInt32) +FIXINT_GETSET(u8, uint8_t, 8, PyLong_FromUInt32) +FIXINT_GETSET(i16, int16_t, 16, PyLong_FromInt32) +FIXINT_GETSET_SW(i16, int16_t, 16, PyLong_FromInt32, _Py_bswap16) +FIXINT_GETSET(u16, uint16_t, 16, PyLong_FromUInt32) +FIXINT_GETSET_SW(u16, uint16_t, 16, PyLong_FromUInt32, _Py_bswap16) +FIXINT_GETSET(i32, int32_t, 32, PyLong_FromInt32) +FIXINT_GETSET_SW(i32, int32_t, 32, PyLong_FromInt32, _Py_bswap32) +FIXINT_GETSET(u32, uint32_t, 32, PyLong_FromUInt32) +FIXINT_GETSET_SW(u32, uint32_t, 32, PyLong_FromUInt32, _Py_bswap32) +FIXINT_GETSET(i64, int64_t, 64, PyLong_FromInt64) +FIXINT_GETSET_SW(i64, int64_t, 64, PyLong_FromInt64, _Py_bswap64) +FIXINT_GETSET(u64, uint64_t, 64, PyLong_FromUInt64) +FIXINT_GETSET_SW(u64, uint64_t, 64, PyLong_FromUInt64, _Py_bswap64) +/*[python end generated code: output=3d60c96fa58e07d5 input=0b7e166f2ea18e70]*/ + +// For one-byte types, swapped variants are the same as native +#define i8_set_sw i8_set +#define i8_get_sw i8_get +#define u8_set_sw u8_set +#define u8_get_sw u8_get + +#undef FIXINT_GETSET +#undef FIXINT_GETSET_SW -static PyObject * -i_get(void *ptr, Py_ssize_t size) -{ - int val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -i_get_sw(void *ptr, Py_ssize_t size) -{ - int val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_INT(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} +/***************************************************************** + * non-integer accessor methods, not supporting bit fields + */ #ifndef MS_WIN32 /* http://msdn.microsoft.com/en-us/library/cc237864.aspx */ #define VARIANT_FALSE 0x0000 #define VARIANT_TRUE 0xFFFF #endif -/* short BOOL - VARIANT_BOOL */ +/* v: short BOOL - VARIANT_BOOL */ static PyObject * -vBOOL_set(void *ptr, PyObject *value, Py_ssize_t size) +v_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(short int))); switch (PyObject_IsTrue(value)) { case -1: return NULL; @@ -664,22 +526,25 @@ vBOOL_set(void *ptr, PyObject *value, Py_ssize_t size) } static PyObject * -vBOOL_get(void *ptr, Py_ssize_t size) +v_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(short int))); return PyBool_FromLong((long)*(short int *)ptr); } +/* bool ('?'): bool (i.e. _Bool) */ static PyObject * bool_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(bool))); switch (PyObject_IsTrue(value)) { case -1: return NULL; case 0: - *(_Bool *)ptr = 0; + *(bool *)ptr = 0; _RET(value); default: - *(_Bool *)ptr = 1; + *(bool *)ptr = 1; _RET(value); } } @@ -687,260 +552,15 @@ bool_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * bool_get(void *ptr, Py_ssize_t size) { - return PyBool_FromLong((long)*(_Bool *)ptr); -} - -static PyObject * -I_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned int x; - if (get_ulong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(unsigned int, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -I_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned int field; - if (get_ulong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_INT(field); - field = SET(unsigned int, field, (unsigned int)val, size); - field = SWAP_INT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - - -static PyObject * -I_get(void *ptr, Py_ssize_t size) -{ - unsigned int val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); -} - -static PyObject * -I_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned int val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_INT(val); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); -} - -static PyObject * -l_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - long x; - if (get_long(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -l_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - long field; - if (get_long(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG(field); - field = SET(long, field, val, size); - field = SWAP_LONG(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - - -static PyObject * -l_get(void *ptr, Py_ssize_t size) -{ - long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -l_get_sw(void *ptr, Py_ssize_t size) -{ - long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -L_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned long x; - if (get_ulong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(unsigned long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -L_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned long field; - if (get_ulong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG(field); - field = SET(unsigned long, field, val, size); - field = SWAP_LONG(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - - -static PyObject * -L_get(void *ptr, Py_ssize_t size) -{ - unsigned long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); -} - -static PyObject * -L_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); -} - -static PyObject * -q_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long long val; - long long x; - if (get_longlong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(long long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -q_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long long val; - long long field; - if (get_longlong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG_LONG(field); - field = SET(long long, field, val, size); - field = SWAP_LONG_LONG(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - -static PyObject * -q_get(void *ptr, Py_ssize_t size) -{ - long long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLongLong(val); -} - -static PyObject * -q_get_sw(void *ptr, Py_ssize_t size) -{ - long long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromLongLong(val); -} - -static PyObject * -Q_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long long val; - unsigned long long x; - if (get_ulonglong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(long long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -Q_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long long val; - unsigned long long field; - if (get_ulonglong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG_LONG(field); - field = SET(unsigned long long, field, val, size); - field = SWAP_LONG_LONG(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - -static PyObject * -Q_get(void *ptr, Py_ssize_t size) -{ - unsigned long long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLongLong(val); -} - -static PyObject * -Q_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned long long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLongLong(val); + assert(NUM_BITS(size) || (size == sizeof(bool))); + return PyBool_FromLong((long)*(bool *)ptr); } -/***************************************************************** - * non-integer accessor methods, not supporting bit fields - */ - - +/* g: long double */ static PyObject * g_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(long double))); long double x; x = PyFloat_AsDouble(value); @@ -953,14 +573,17 @@ g_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * g_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(long double))); long double val; memcpy(&val, ptr, sizeof(long double)); return PyFloat_FromDouble(val); } +/* d: double */ static PyObject * d_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double))); double x; x = PyFloat_AsDouble(value); @@ -973,15 +596,18 @@ d_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * d_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double))); double val; memcpy(&val, ptr, sizeof(val)); return PyFloat_FromDouble(val); } #if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +/* C: double complex */ static PyObject * C_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double complex))); Py_complex c = PyComplex_AsCComplex(value); if (c.real == -1 && PyErr_Occurred()) { @@ -995,15 +621,18 @@ C_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * C_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double complex))); double complex x; memcpy(&x, ptr, sizeof(x)); return PyComplex_FromDoubles(creal(x), cimag(x)); } +/* E: float complex */ static PyObject * E_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float complex))); Py_complex c = PyComplex_AsCComplex(value); if (c.real == -1 && PyErr_Occurred()) { @@ -1017,15 +646,18 @@ E_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * E_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float complex))); float complex x; memcpy(&x, ptr, sizeof(x)); return PyComplex_FromDoubles(crealf(x), cimagf(x)); } +/* F: long double complex */ static PyObject * F_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(long double complex))); Py_complex c = PyComplex_AsCComplex(value); if (c.real == -1 && PyErr_Occurred()) { @@ -1039,6 +671,7 @@ F_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * F_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(long double complex))); long double complex x; memcpy(&x, ptr, sizeof(x)); @@ -1046,9 +679,11 @@ F_get(void *ptr, Py_ssize_t size) } #endif +/* d: double */ static PyObject * d_set_sw(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double))); double x; x = PyFloat_AsDouble(value); @@ -1067,6 +702,7 @@ d_set_sw(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * d_get_sw(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double))); #ifdef WORDS_BIGENDIAN return PyFloat_FromDouble(PyFloat_Unpack8(ptr, 1)); #else @@ -1074,9 +710,11 @@ d_get_sw(void *ptr, Py_ssize_t size) #endif } +/* f: float */ static PyObject * f_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); float x; x = (float)PyFloat_AsDouble(value); @@ -1089,6 +727,7 @@ f_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * f_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); float val; memcpy(&val, ptr, sizeof(val)); return PyFloat_FromDouble(val); @@ -1097,6 +736,7 @@ f_get(void *ptr, Py_ssize_t size) static PyObject * f_set_sw(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); float x; x = (float)PyFloat_AsDouble(value); @@ -1115,6 +755,7 @@ f_set_sw(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * f_get_sw(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); #ifdef WORDS_BIGENDIAN return PyFloat_FromDouble(PyFloat_Unpack4(ptr, 1)); #else @@ -1122,6 +763,7 @@ f_get_sw(void *ptr, Py_ssize_t size) #endif } +/* O: Python object */ /* py_object refcounts: @@ -1135,6 +777,7 @@ f_get_sw(void *ptr, Py_ssize_t size) static PyObject * O_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(PyObject *))); PyObject *ob = *(PyObject **)ptr; if (ob == NULL) { if (!PyErr_Occurred()) @@ -1149,15 +792,18 @@ O_get(void *ptr, Py_ssize_t size) static PyObject * O_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(PyObject *))); /* Hm, does the memory block need it's own refcount or not? */ *(PyObject **)ptr = value; return Py_NewRef(value); } +/* c: a single byte-character */ static PyObject * c_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(char))); if (PyBytes_Check(value)) { if (PyBytes_GET_SIZE(value) != 1) { PyErr_Format(PyExc_TypeError, @@ -1204,13 +850,15 @@ c_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * c_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(char))); return PyBytes_FromStringAndSize((char *)ptr, 1); } -/* u - a single wchar_t character */ +/* u: a single wchar_t character */ static PyObject * u_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(wchar_t))); Py_ssize_t len; wchar_t chars[2]; if (!PyUnicode_Check(value)) { @@ -1244,10 +892,11 @@ u_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * u_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(wchar_t))); return PyUnicode_FromWideChar((wchar_t *)ptr, 1); } -/* U - a unicode string */ +/* U: a wchar_t* unicode string */ static PyObject * U_get(void *ptr, Py_ssize_t size) { @@ -1306,6 +955,7 @@ U_set(void *ptr, PyObject *value, Py_ssize_t length) } +/* s: a byte string */ static PyObject * s_get(void *ptr, Py_ssize_t size) { @@ -1355,6 +1005,7 @@ s_set(void *ptr, PyObject *value, Py_ssize_t length) _RET(value); } +/* z: a byte string, can be set from integer pointer */ static PyObject * z_set(void *ptr, PyObject *value, Py_ssize_t size) { @@ -1391,6 +1042,7 @@ z_get(void *ptr, Py_ssize_t size) } } +/* Z: a wchar* string, can be set from integer pointer */ static PyObject * Z_set(void *ptr, PyObject *value, Py_ssize_t size) { @@ -1445,8 +1097,9 @@ Z_get(void *ptr, Py_ssize_t size) #ifdef MS_WIN32 +/* X: COM BSTR (wide-char string to be handled handled using Windows API) */ static PyObject * -BSTR_set(void *ptr, PyObject *value, Py_ssize_t size) +X_set(void *ptr, PyObject *value, Py_ssize_t size) { BSTR bstr; @@ -1490,7 +1143,7 @@ BSTR_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * -BSTR_get(void *ptr, Py_ssize_t size) +X_get(void *ptr, Py_ssize_t size) { BSTR p; p = *(BSTR *)ptr; @@ -1505,9 +1158,11 @@ BSTR_get(void *ptr, Py_ssize_t size) } #endif +/* P: generic pointer */ static PyObject * P_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(void *))); void *v; if (value == Py_None) { *(void **)ptr = NULL; @@ -1539,154 +1194,339 @@ P_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * P_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(void *))); if (*(void **)ptr == NULL) { Py_RETURN_NONE; } return PyLong_FromVoidPtr(*(void **)ptr); } -static struct fielddesc formattable[] = { - { 's', s_set, s_get, NULL}, - { 'b', b_set, b_get, NULL}, - { 'B', B_set, B_get, NULL}, - { 'c', c_set, c_get, NULL}, - { 'd', d_set, d_get, NULL, d_set_sw, d_get_sw}, -#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) - { 'C', C_set, C_get, NULL}, - { 'E', E_set, E_get, NULL}, - { 'F', F_set, F_get, NULL}, -#endif - { 'g', g_set, g_get, NULL}, - { 'f', f_set, f_get, NULL, f_set_sw, f_get_sw}, - { 'h', h_set, h_get, NULL, h_set_sw, h_get_sw}, - { 'H', H_set, H_get, NULL, H_set_sw, H_get_sw}, - { 'i', i_set, i_get, NULL, i_set_sw, i_get_sw}, - { 'I', I_set, I_get, NULL, I_set_sw, I_get_sw}, - { 'l', l_set, l_get, NULL, l_set_sw, l_get_sw}, - { 'L', L_set, L_get, NULL, L_set_sw, L_get_sw}, - { 'q', q_set, q_get, NULL, q_set_sw, q_get_sw}, - { 'Q', Q_set, Q_get, NULL, Q_set_sw, Q_get_sw}, - { 'P', P_set, P_get, NULL}, - { 'z', z_set, z_get, NULL}, - { 'u', u_set, u_get, NULL}, - { 'U', U_set, U_get, NULL}, - { 'Z', Z_set, Z_get, NULL}, -#ifdef MS_WIN32 - { 'X', BSTR_set, BSTR_get, NULL}, -#endif - { 'v', vBOOL_set, vBOOL_get, NULL}, -#if SIZEOF__BOOL == SIZEOF_INT - { '?', bool_set, bool_get, NULL, I_set_sw, I_get_sw}, -#elif SIZEOF__BOOL == SIZEOF_LONG - { '?', bool_set, bool_get, NULL, L_set_sw, L_get_sw}, -#elif SIZEOF__BOOL == SIZEOF_LONG_LONG - { '?', bool_set, bool_get, NULL, Q_set_sw, Q_get_sw}, -#else - { '?', bool_set, bool_get, NULL}, -#endif /* SIZEOF__BOOL */ - { 'O', O_set, O_get, NULL}, - { 0, NULL, NULL, NULL}, +/* Table with info about all formats. + * Must be accessed via _ctypes_get_fielddesc, which initializes it on + * first use. After initialization it's treated as constant & read-only. + */ + +struct formattable { +/*[python input] +for nbytes in 8, 16, 32, 64: + for sgn in 'i', 'u': + print(f' struct fielddesc fmt_{sgn}{nbytes};') +for code in 'sbBcdCEFgfhHiIlLqQPzuUZXvO': + print(f' struct fielddesc fmt_{code};') +[python start generated code]*/ + struct fielddesc fmt_i8; + struct fielddesc fmt_u8; + struct fielddesc fmt_i16; + struct fielddesc fmt_u16; + struct fielddesc fmt_i32; + struct fielddesc fmt_u32; + struct fielddesc fmt_i64; + struct fielddesc fmt_u64; + struct fielddesc fmt_s; + struct fielddesc fmt_b; + struct fielddesc fmt_B; + struct fielddesc fmt_c; + struct fielddesc fmt_d; + struct fielddesc fmt_C; + struct fielddesc fmt_E; + struct fielddesc fmt_F; + struct fielddesc fmt_g; + struct fielddesc fmt_f; + struct fielddesc fmt_h; + struct fielddesc fmt_H; + struct fielddesc fmt_i; + struct fielddesc fmt_I; + struct fielddesc fmt_l; + struct fielddesc fmt_L; + struct fielddesc fmt_q; + struct fielddesc fmt_Q; + struct fielddesc fmt_P; + struct fielddesc fmt_z; + struct fielddesc fmt_u; + struct fielddesc fmt_U; + struct fielddesc fmt_Z; + struct fielddesc fmt_X; + struct fielddesc fmt_v; + struct fielddesc fmt_O; +/*[python end generated code: output=fa648744ec7f919d input=087d58357d4bf2c5]*/ + + // bool has code '?': + struct fielddesc fmt_bool; + + // always contains NULLs: + struct fielddesc fmt_nil; }; -/* - Ideas: Implement VARIANT in this table, using 'V' code. - Use '?' as code for BOOL. -*/ +static struct formattable formattable; + + +/* Get fielddesc info for a fixed-width integer. + * N.B: - must be called after (or from) _ctypes_init_fielddesc! + * - nbytes must be one of the supported values + */ + +static inline struct fielddesc * +_ctypes_fixint_fielddesc(Py_ssize_t nbytes, bool is_signed) +{ +#define _PACK(NBYTES, SGN) ((NBYTES<<2) + (SGN ? 1 : 0)) + switch (_PACK(nbytes, is_signed)) { +/*[python input] +for nbytes in 8, 16, 32, 64: + for sgn in 'i', 'u': + is_signed = sgn == 'i' + print(f' case (_PACK({nbytes // 8}, {int(is_signed)})): ' + + f'return &formattable.fmt_{sgn}{nbytes};') +[python start generated code]*/ + case (_PACK(1, 1)): return &formattable.fmt_i8; + case (_PACK(1, 0)): return &formattable.fmt_u8; + case (_PACK(2, 1)): return &formattable.fmt_i16; + case (_PACK(2, 0)): return &formattable.fmt_u16; + case (_PACK(4, 1)): return &formattable.fmt_i32; + case (_PACK(4, 0)): return &formattable.fmt_u32; + case (_PACK(8, 1)): return &formattable.fmt_i64; + case (_PACK(8, 0)): return &formattable.fmt_u64; +/*[python end generated code: output=0194ba35c4d64ff3 input=ee9f6f5bb872d645]*/ +#undef _PACK + } + /* ctypes currently only supports platforms where the basic integer types + * (`char`, `short`, `int`, `long`, `long long`) have 1, 2, 4, or 8 bytes + * (i.e. 8 to 64 bits). + */ + Py_UNREACHABLE(); +} + + +/* Macro to call _ctypes_fixint_fielddesc for a given C type. */ + +_Py_COMP_DIAG_PUSH +#if defined(__GNUC__) && (__GNUC__ < 14) +/* The signedness check expands to an expression that's always true or false. + * Older GCC gives a '-Wtype-limits' warning for this, which is a GCC bug + * (docs say it should "not warn for constant expressions"): + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86647 + * Silence that warning. + */ +#pragma GCC diagnostic ignored "-Wtype-limits" +#endif + +#define FIXINT_FIELDDESC_FOR(C_TYPE) \ + _ctypes_fixint_fielddesc(sizeof(C_TYPE), (C_TYPE)-1 < 0) + /* Delayed initialization. Windows cannot statically reference dynamically loaded addresses from DLLs. */ void _ctypes_init_fielddesc(void) { - struct fielddesc *fd = formattable; - for (; fd->code; ++fd) { - switch (fd->code) { - case 's': fd->pffi_type = &ffi_type_pointer; break; - case 'b': fd->pffi_type = &ffi_type_schar; break; - case 'B': fd->pffi_type = &ffi_type_uchar; break; - case 'c': fd->pffi_type = &ffi_type_schar; break; - case 'd': fd->pffi_type = &ffi_type_double; break; + /* Fixed-width integers */ + +/*[python input] +for nbytes in 8, 16, 32, 64: + for sgn in 'i', 'u': + is_signed = sgn == 'i' + u = 'u' if sgn == 'u' else 's' + parts = [ + f"0", + f'&ffi_type_{u}int{nbytes}', + f'{sgn}{nbytes}_set', + f'{sgn}{nbytes}_get', + f'{sgn}{nbytes}_set_sw', + f'{sgn}{nbytes}_get_sw', + ] + print(f' formattable.fmt_{sgn}{nbytes} = (struct fielddesc){{') + print(f' {', '.join(parts)} }};') +[python start generated code]*/ + formattable.fmt_i8 = (struct fielddesc){ + 0, &ffi_type_sint8, i8_set, i8_get, i8_set_sw, i8_get_sw }; + formattable.fmt_u8 = (struct fielddesc){ + 0, &ffi_type_uint8, u8_set, u8_get, u8_set_sw, u8_get_sw }; + formattable.fmt_i16 = (struct fielddesc){ + 0, &ffi_type_sint16, i16_set, i16_get, i16_set_sw, i16_get_sw }; + formattable.fmt_u16 = (struct fielddesc){ + 0, &ffi_type_uint16, u16_set, u16_get, u16_set_sw, u16_get_sw }; + formattable.fmt_i32 = (struct fielddesc){ + 0, &ffi_type_sint32, i32_set, i32_get, i32_set_sw, i32_get_sw }; + formattable.fmt_u32 = (struct fielddesc){ + 0, &ffi_type_uint32, u32_set, u32_get, u32_set_sw, u32_get_sw }; + formattable.fmt_i64 = (struct fielddesc){ + 0, &ffi_type_sint64, i64_set, i64_get, i64_set_sw, i64_get_sw }; + formattable.fmt_u64 = (struct fielddesc){ + 0, &ffi_type_uint64, u64_set, u64_get, u64_set_sw, u64_get_sw }; +/*[python end generated code: output=16806fe0ca3a9c4c input=850b8dd6388b1b10]*/ + + + /* Native C integers. + * These use getters/setters for fixed-width ints but have their own + * `code` and `pffi_type`. + */ + +/*[python input] +for base_code, base_c_type in [ + ('b', 'char'), + ('h', 'short'), + ('i', 'int'), + ('l', 'long'), + ('q', 'long long'), +]: + for code, c_type, ffi_type in [ + (base_code, 'signed ' + base_c_type, 's' + base_c_type), + (base_code.upper(), 'unsigned ' + base_c_type, 'u' + base_c_type), + ]: + print(f' formattable.fmt_{code} = *FIXINT_FIELDDESC_FOR({c_type});') + print(f" formattable.fmt_{code}.code = '{code}';") + if base_code == 'q': + # ffi doesn't have `long long`; keep use the fixint type + pass + else: + print(f' formattable.fmt_{code}.pffi_type = &ffi_type_{ffi_type};') +[python start generated code]*/ + formattable.fmt_b = *FIXINT_FIELDDESC_FOR(signed char); + formattable.fmt_b.code = 'b'; + formattable.fmt_b.pffi_type = &ffi_type_schar; + formattable.fmt_B = *FIXINT_FIELDDESC_FOR(unsigned char); + formattable.fmt_B.code = 'B'; + formattable.fmt_B.pffi_type = &ffi_type_uchar; + formattable.fmt_h = *FIXINT_FIELDDESC_FOR(signed short); + formattable.fmt_h.code = 'h'; + formattable.fmt_h.pffi_type = &ffi_type_sshort; + formattable.fmt_H = *FIXINT_FIELDDESC_FOR(unsigned short); + formattable.fmt_H.code = 'H'; + formattable.fmt_H.pffi_type = &ffi_type_ushort; + formattable.fmt_i = *FIXINT_FIELDDESC_FOR(signed int); + formattable.fmt_i.code = 'i'; + formattable.fmt_i.pffi_type = &ffi_type_sint; + formattable.fmt_I = *FIXINT_FIELDDESC_FOR(unsigned int); + formattable.fmt_I.code = 'I'; + formattable.fmt_I.pffi_type = &ffi_type_uint; + formattable.fmt_l = *FIXINT_FIELDDESC_FOR(signed long); + formattable.fmt_l.code = 'l'; + formattable.fmt_l.pffi_type = &ffi_type_slong; + formattable.fmt_L = *FIXINT_FIELDDESC_FOR(unsigned long); + formattable.fmt_L.code = 'L'; + formattable.fmt_L.pffi_type = &ffi_type_ulong; + formattable.fmt_q = *FIXINT_FIELDDESC_FOR(signed long long); + formattable.fmt_q.code = 'q'; + formattable.fmt_Q = *FIXINT_FIELDDESC_FOR(unsigned long long); + formattable.fmt_Q.code = 'Q'; +/*[python end generated code: output=873c87a2e6b5075a input=ee814ca263aac18e]*/ + + + /* Other types have bespoke setters and getters named `@_set` and `@_get`, + * where `@` is the type code. + * Some have swapped variants, `@_set_sw` and `@_get_sw` + */ + +#define _TABLE_ENTRY(SYMBOL, FFI_TYPE, ...) \ + formattable.fmt_ ## SYMBOL = \ + (struct fielddesc){(#SYMBOL)[0], (FFI_TYPE), __VA_ARGS__}; \ + /////////////////////////////////////////////////////////////////////////// + +#define TABLE_ENTRY(SYMBOL, FFI_TYPE) \ + _TABLE_ENTRY(SYMBOL, FFI_TYPE, SYMBOL ## _set, SYMBOL ## _get) \ + /////////////////////////////////////////////////////////////////////////// + +#define TABLE_ENTRY_SW(SYMBOL, FFI_TYPE) \ + _TABLE_ENTRY(SYMBOL, FFI_TYPE, SYMBOL ## _set, \ + SYMBOL ## _get, SYMBOL ## _set_sw, SYMBOL ## _get_sw) \ + /////////////////////////////////////////////////////////////////////////// + + TABLE_ENTRY_SW(d, &ffi_type_double); #if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) - case 'C': fd->pffi_type = &ffi_type_complex_double; break; - case 'E': fd->pffi_type = &ffi_type_complex_float; break; - case 'F': fd->pffi_type = &ffi_type_complex_longdouble; break; + TABLE_ENTRY(C, &ffi_type_complex_double); + TABLE_ENTRY(E, &ffi_type_complex_float); + TABLE_ENTRY(F, &ffi_type_complex_longdouble); #endif - case 'g': fd->pffi_type = &ffi_type_longdouble; break; - case 'f': fd->pffi_type = &ffi_type_float; break; - case 'h': fd->pffi_type = &ffi_type_sshort; break; - case 'H': fd->pffi_type = &ffi_type_ushort; break; - case 'i': fd->pffi_type = &ffi_type_sint; break; - case 'I': fd->pffi_type = &ffi_type_uint; break; - /* XXX Hm, sizeof(int) == sizeof(long) doesn't hold on every platform */ - /* As soon as we can get rid of the type codes, this is no longer a problem */ - #if SIZEOF_LONG == 4 - case 'l': fd->pffi_type = &ffi_type_sint32; break; - case 'L': fd->pffi_type = &ffi_type_uint32; break; - #elif SIZEOF_LONG == 8 - case 'l': fd->pffi_type = &ffi_type_sint64; break; - case 'L': fd->pffi_type = &ffi_type_uint64; break; - #else - #error - #endif - #if SIZEOF_LONG_LONG == 8 - case 'q': fd->pffi_type = &ffi_type_sint64; break; - case 'Q': fd->pffi_type = &ffi_type_uint64; break; - #else - #error - #endif - case 'P': fd->pffi_type = &ffi_type_pointer; break; - case 'z': fd->pffi_type = &ffi_type_pointer; break; - case 'u': - if (sizeof(wchar_t) == sizeof(short)) - fd->pffi_type = &ffi_type_sshort; - else if (sizeof(wchar_t) == sizeof(int)) - fd->pffi_type = &ffi_type_sint; - else if (sizeof(wchar_t) == sizeof(long)) - fd->pffi_type = &ffi_type_slong; - else - Py_UNREACHABLE(); - break; - case 'U': fd->pffi_type = &ffi_type_pointer; break; - case 'Z': fd->pffi_type = &ffi_type_pointer; break; - #ifdef MS_WIN32 - case 'X': fd->pffi_type = &ffi_type_pointer; break; - #endif - case 'v': fd->pffi_type = &ffi_type_sshort; break; - #if SIZEOF__BOOL == 1 - case '?': fd->pffi_type = &ffi_type_uchar; break; /* Also fallback for no native _Bool support */ - #elif SIZEOF__BOOL == SIZEOF_SHORT - case '?': fd->pffi_type = &ffi_type_ushort; break; - #elif SIZEOF__BOOL == SIZEOF_INT - case '?': fd->pffi_type = &ffi_type_uint; break; - #elif SIZEOF__BOOL == SIZEOF_LONG - case '?': fd->pffi_type = &ffi_type_ulong; break; - #elif SIZEOF__BOOL == SIZEOF_LONG_LONG - case '?': fd->pffi_type = &ffi_type_ulong; break; - #endif /* SIZEOF__BOOL */ - case 'O': fd->pffi_type = &ffi_type_pointer; break; - default: - Py_UNREACHABLE(); - } - } + TABLE_ENTRY(g, &ffi_type_longdouble); + TABLE_ENTRY_SW(f, &ffi_type_float); + TABLE_ENTRY(v, &ffi_type_sshort); /* vBOOL */ + + // ctypes.c_char is signed for FFI, even where C wchar_t is unsigned. + TABLE_ENTRY(c, _ctypes_fixint_fielddesc(sizeof(char), true)->pffi_type); + // ctypes.c_wchar is signed for FFI, even where C wchar_t is unsigned. + TABLE_ENTRY(u, _ctypes_fixint_fielddesc(sizeof(wchar_t), true)->pffi_type); + + TABLE_ENTRY(s, &ffi_type_pointer); + TABLE_ENTRY(P, &ffi_type_pointer); + TABLE_ENTRY(z, &ffi_type_pointer); + TABLE_ENTRY(U, &ffi_type_pointer); + TABLE_ENTRY(Z, &ffi_type_pointer); +#ifdef MS_WIN32 + TABLE_ENTRY(X, &ffi_type_pointer); +#endif + TABLE_ENTRY(O, &ffi_type_pointer); + +#undef TABLE_ENTRY_SW +#undef TABLE_ENTRY +#undef _TABLE_ENTRY + /* bool has code '?', fill it in manually */ + + // ctypes.c_bool is unsigned for FFI, even where C bool is signed. + formattable.fmt_bool = *_ctypes_fixint_fielddesc(sizeof(bool), false); + formattable.fmt_bool.code = '?'; + formattable.fmt_bool.setfunc = bool_set; + formattable.fmt_bool.getfunc = bool_get; } +#undef FIXINT_FIELDDESC_FOR +_Py_COMP_DIAG_POP struct fielddesc * _ctypes_get_fielddesc(const char *fmt) { - static int initialized = 0; - struct fielddesc *table = formattable; - + static bool initialized = false; + static PyMutex mutex = {0}; + PyMutex_Lock(&mutex); if (!initialized) { - initialized = 1; _ctypes_init_fielddesc(); + initialized = true; } - - for (; table->code; ++table) { - if (table->code == fmt[0]) - return table; + PyMutex_Unlock(&mutex); + struct fielddesc *result = NULL; + switch(fmt[0]) { +/*[python input] +for code in 'sbBcdCEFgfhHiIlLqQPzuUZXvO': + print(f" case '{code}': result = &formattable.fmt_{code}; break;") +[python start generated code]*/ + case 's': result = &formattable.fmt_s; break; + case 'b': result = &formattable.fmt_b; break; + case 'B': result = &formattable.fmt_B; break; + case 'c': result = &formattable.fmt_c; break; + case 'd': result = &formattable.fmt_d; break; + case 'C': result = &formattable.fmt_C; break; + case 'E': result = &formattable.fmt_E; break; + case 'F': result = &formattable.fmt_F; break; + case 'g': result = &formattable.fmt_g; break; + case 'f': result = &formattable.fmt_f; break; + case 'h': result = &formattable.fmt_h; break; + case 'H': result = &formattable.fmt_H; break; + case 'i': result = &formattable.fmt_i; break; + case 'I': result = &formattable.fmt_I; break; + case 'l': result = &formattable.fmt_l; break; + case 'L': result = &formattable.fmt_L; break; + case 'q': result = &formattable.fmt_q; break; + case 'Q': result = &formattable.fmt_Q; break; + case 'P': result = &formattable.fmt_P; break; + case 'z': result = &formattable.fmt_z; break; + case 'u': result = &formattable.fmt_u; break; + case 'U': result = &formattable.fmt_U; break; + case 'Z': result = &formattable.fmt_Z; break; + case 'X': result = &formattable.fmt_X; break; + case 'v': result = &formattable.fmt_v; break; + case 'O': result = &formattable.fmt_O; break; +/*[python end generated code: output=81a8223dda9f81f7 input=2f59666d3c024edf]*/ + case '?': result = &formattable.fmt_bool; break; } - return NULL; + if (!result || !result->code) { + return NULL; + } + assert(result->pffi_type); + assert(result->setfunc); + assert(result->getfunc); + return result; } +/* + Ideas: Implement VARIANT in this table, using 'V' code. +*/ + /*---------------- EOF ----------------*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7e0804054cded4..45e00a538fb5a5 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -113,8 +113,17 @@ extern PyType_Spec cthunk_spec; typedef struct tagPyCArgObject PyCArgObject; typedef struct tagCDataObject CDataObject; -typedef PyObject *(* GETFUNC)(void *, Py_ssize_t size); -typedef PyObject *(* SETFUNC)(void *, PyObject *value, Py_ssize_t size); + +// GETFUNC: convert the C value at *ptr* to Python object, return the object +// SETFUNC: write content of the PyObject *value* to the location at *ptr*; +// return a new reference to either *value*, or None for simple types +// (see _CTYPES_DEBUG_KEEP). +// Note that the *size* arg can have different meanings depending on context: +// for string-like arrays it's the size in bytes +// for int-style fields it's either the type size, or bitfiled info +// that can be unpacked using the LOW_BIT & NUM_BITS macros. +typedef PyObject *(* GETFUNC)(void *ptr, Py_ssize_t size); +typedef PyObject *(* SETFUNC)(void *ptr, PyObject *value, Py_ssize_t size); typedef PyCArgObject *(* PARAMFUNC)(ctypes_state *st, CDataObject *obj); /* A default buffer in CDataObject, which can be used for small C types. If @@ -239,9 +248,9 @@ extern CThunkObject *_ctypes_alloc_callback(ctypes_state *st, /* a table entry describing a predefined ctypes type */ struct fielddesc { char code; + ffi_type *pffi_type; /* always statically allocated */ SETFUNC setfunc; GETFUNC getfunc; - ffi_type *pffi_type; /* always statically allocated */ SETFUNC setfunc_swapped; GETFUNC getfunc_swapped; }; From 128cc47fbd44e3e09c50d9674fe4a4bba3be450c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Dec 2024 16:52:20 +0000 Subject: [PATCH 260/427] GH-127705: Add debug mode for `_PyStackRef`s inspired by HPy debug mode (GH-128121) --- Include/internal/pycore_interp.h | 6 + Include/internal/pycore_stackref.h | 113 +++++++++++++ Makefile.pre.in | 1 + ...-12-20-12-25-16.gh-issue-127705.WmCz1z.rst | 4 + Objects/frameobject.c | 4 +- Python/bytecodes.c | 17 +- Python/ceval.c | 67 ++++++-- Python/ceval_macros.h | 6 +- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 23 ++- Python/pystate.c | 29 ++++ Python/stackrefs.c | 156 ++++++++++++++++++ 12 files changed, 395 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst create mode 100644 Python/stackrefs.c diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 87cdcb5b119d15..a3c14dceffd7a0 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -34,6 +34,7 @@ extern "C" { #include "pycore_optimizer.h" // _PyOptimizerObject #include "pycore_obmalloc.h" // struct _obmalloc_state #include "pycore_qsbr.h" // struct _qsbr_state +#include "pycore_stackref.h" // Py_STACKREF_DEBUG #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_uniqueid.h" // struct _Py_unique_id_pool @@ -285,6 +286,11 @@ struct _is { _PyThreadStateImpl _initial_thread; // _initial_thread should be the last field of PyInterpreterState. // See https://github.com/python/cpython/issues/127117. + +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + uint64_t next_stackref; + _Py_hashtable_t *stackref_debug_table; +#endif }; diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 90a3118352f7ae..1ae62cc69bb364 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -4,6 +4,9 @@ extern "C" { #endif +// Define this to get precise tracking of stackrefs. +// #define Py_STACKREF_DEBUG 1 + #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif @@ -49,6 +52,113 @@ extern "C" { CPython refcounting operations on it! */ + +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + + + +typedef union _PyStackRef { + uint64_t index; +} _PyStackRef; + +#define Py_TAG_BITS 0 + +PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref); +PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref); +PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, const char *filename, int linenumber); +PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber); +extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); + +static const _PyStackRef PyStackRef_NULL = { .index = 0 }; + +#define PyStackRef_None ((_PyStackRef){ .index = 1 } ) +#define PyStackRef_False ((_PyStackRef){ .index = 2 }) +#define PyStackRef_True ((_PyStackRef){ .index = 3 }) + +#define LAST_PREDEFINED_STACKREF_INDEX 3 + +static inline int +PyStackRef_IsNull(_PyStackRef ref) +{ + return ref.index == 0; +} + +static inline int +PyStackRef_IsTrue(_PyStackRef ref) +{ + return _Py_stackref_get_object(ref) == Py_True; +} + +static inline int +PyStackRef_IsFalse(_PyStackRef ref) +{ + return _Py_stackref_get_object(ref) == Py_False; +} + +static inline int +PyStackRef_IsNone(_PyStackRef ref) +{ + return _Py_stackref_get_object(ref) == Py_None; +} + +static inline PyObject * +_PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber) +{ + _Py_stackref_record_borrow(ref, filename, linenumber); + return _Py_stackref_get_object(ref); +} + +#define PyStackRef_AsPyObjectBorrow(REF) _PyStackRef_AsPyObjectBorrow((REF), __FILE__, __LINE__) + +static inline PyObject * +PyStackRef_AsPyObjectSteal(_PyStackRef ref) +{ + return _Py_stackref_close(ref); +} + +static inline _PyStackRef +_PyStackRef_FromPyObjectNew(PyObject *obj, const char *filename, int linenumber) +{ + Py_INCREF(obj); + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj), __FILE__, __LINE__) + +static inline _PyStackRef +_PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumber) +{ + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__) + +static inline _PyStackRef +_PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenumber) +{ + assert(_Py_IsImmortal(obj)); + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__) + +static inline void +PyStackRef_CLOSE(_PyStackRef ref) +{ + PyObject *obj = _Py_stackref_close(ref); + Py_DECREF(obj); +} + +static inline _PyStackRef +_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) +{ + PyObject *obj = _Py_stackref_get_object(ref); + Py_INCREF(obj); + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) + +#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) PyStackRef_CLOSE(stackref) + +#else + typedef union _PyStackRef { uintptr_t bits; } _PyStackRef; @@ -200,12 +310,15 @@ static const _PyStackRef PyStackRef_NULL = { .bits = 0 }; #define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True) #define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False) +#endif + // Converts a PyStackRef back to a PyObject *, converting the // stackref to a new reference. #define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)) #define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) + #define PyStackRef_CLEAR(op) \ do { \ _PyStackRef *_tmp_op_ptr = &(op); \ diff --git a/Makefile.pre.in b/Makefile.pre.in index 3e880f7800fccf..67acf0fc520087 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -488,6 +488,7 @@ PYTHON_OBJS= \ Python/qsbr.o \ Python/bootstrap_hash.o \ Python/specialize.o \ + Python/stackrefs.o \ Python/structmember.o \ Python/symtable.o \ Python/sysmodule.o \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst new file mode 100644 index 00000000000000..fde12b78ce0444 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-12-25-16.gh-issue-127705.WmCz1z.rst @@ -0,0 +1,4 @@ +Adds stackref debugging when ``Py_STACKREF_DEBUG`` is set. Finds all +double-closes and leaks, logging the origin and last borrow. + +Inspired by HPy's debug mode. https://docs.hpyproject.org/en/latest/debug-mode.html diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 03ed2b9480f8c9..10fd3a982c36f4 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -179,9 +179,9 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) if (kind == CO_FAST_FREE) { // The cell was set when the frame was created from // the function's closure. - assert(oldvalue.bits != 0 && PyCell_Check(PyStackRef_AsPyObjectBorrow(oldvalue))); + assert(!PyStackRef_IsNull(oldvalue) && PyCell_Check(PyStackRef_AsPyObjectBorrow(oldvalue))); cell = PyStackRef_AsPyObjectBorrow(oldvalue); - } else if (kind & CO_FAST_CELL && oldvalue.bits != 0) { + } else if (kind & CO_FAST_CELL && !PyStackRef_IsNull(oldvalue)) { PyObject *as_obj = PyStackRef_AsPyObjectBorrow(oldvalue); if (PyCell_Check(as_obj)) { cell = as_obj; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 30c12dd4dc9205..63cf1978e8abe5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -681,7 +681,7 @@ dummy_func( assert(Py_REFCNT(left_o) >= 2); PyStackRef_CLOSE(left); DEAD(left); - PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); @@ -4509,17 +4509,17 @@ dummy_func( op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); PyObject *result_o; assert(!_PyErr_Occurred(tstate)); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -4550,7 +4550,10 @@ dummy_func( if (Py_TYPE(func) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); @@ -4568,6 +4571,10 @@ dummy_func( frame->return_offset = 1; DISPATCH_INLINED(new_frame); } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); result_o = PyObject_Call(func, callargs, kwargs); } PyStackRef_XCLOSE(kwargs_st); diff --git a/Python/ceval.c b/Python/ceval.c index bfdf5687c287db..3cf11b663c571b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -164,7 +164,7 @@ dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) PyErr_Clear(); } // Don't call __repr__(), it might recurse into the interpreter. - printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)(ptr->bits)); + printf("<%s at %p>", Py_TYPE(obj)->tp_name, PyStackRef_AsPyObjectBorrow(*ptr)); } printf("]\n"); fflush(stdout); @@ -805,7 +805,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int -#ifdef Py_DEBUG +#if defined(Py_DEBUG) && !defined(Py_STACKREF_DEBUG) /* Set these to invalid but identifiable values for debugging. */ entry_frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; entry_frame.f_locals = (PyObject*)0xaaa1; @@ -1810,27 +1810,48 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, { bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); PyObject *kwnames = NULL; - PyObject *const *newargs; + _PyStackRef *newargs; + PyObject *const *object_array = NULL; + _PyStackRef stack_array[8]; if (has_dict) { - newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); - if (newargs == NULL) { + object_array = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); + if (object_array == NULL) { PyStackRef_CLOSE(func); goto error; } + size_t total_args = nargs + PyDict_GET_SIZE(kwargs); + assert(sizeof(PyObject *) == sizeof(_PyStackRef)); + newargs = (_PyStackRef *)object_array; + for (size_t i = 0; i < total_args; i++) { + newargs[i] = PyStackRef_FromPyObjectSteal(object_array[i]); + } } else { - newargs = &PyTuple_GET_ITEM(callargs, 0); - /* We need to incref all our args since the new frame steals the references. */ - for (Py_ssize_t i = 0; i < nargs; ++i) { - Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + if (nargs <= 8) { + newargs = stack_array; + } + else { + newargs = PyMem_Malloc(sizeof(_PyStackRef) *nargs); + if (newargs == NULL) { + PyErr_NoMemory(); + PyStackRef_CLOSE(func); + goto error; + } + } + /* We need to create a new reference for all our args since the new frame steals them. */ + for (Py_ssize_t i = 0; i < nargs; i++) { + newargs[i] = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(callargs, i)); } } _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, func, locals, - (_PyStackRef const *)newargs, nargs, kwnames, previous + newargs, nargs, kwnames, previous ); if (has_dict) { - _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + _PyStack_UnpackDict_FreeNoDecRef(object_array, kwnames); + } + else if (nargs > 8) { + PyMem_Free((void *)newargs); } /* No need to decref func here because the reference has been stolen by _PyEvalFramePushAndInit. @@ -1850,21 +1871,39 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, PyObject* const* args, size_t argcount, PyObject *kwnames) { + size_t total_args = argcount; + if (kwnames) { + total_args += PyTuple_GET_SIZE(kwnames); + } + _PyStackRef stack_array[8]; + _PyStackRef *arguments; + if (total_args <= 8) { + arguments = stack_array; + } + else { + arguments = PyMem_Malloc(sizeof(_PyStackRef) * total_args); + if (arguments == NULL) { + return PyErr_NoMemory(); + } + } /* _PyEvalFramePushAndInit consumes the references * to func, locals and all its arguments */ Py_XINCREF(locals); for (size_t i = 0; i < argcount; i++) { - Py_INCREF(args[i]); + arguments[i] = PyStackRef_FromPyObjectNew(args[i]); } if (kwnames) { Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); for (Py_ssize_t i = 0; i < kwcount; i++) { - Py_INCREF(args[i+argcount]); + arguments[i+argcount] = PyStackRef_FromPyObjectNew(args[i+argcount]); } } _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( tstate, PyStackRef_FromPyObjectNew(func), locals, - (_PyStackRef const *)args, argcount, kwnames, NULL); + arguments, argcount, kwnames, NULL); + if (total_args > 8) { + PyMem_Free(arguments); + } if (frame == NULL) { return NULL; } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 398816d5f36a1d..f15633fa467376 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -450,7 +450,7 @@ do { \ /* How much scratch space to give stackref to PyObject* conversion. */ #define MAX_STACKREF_SCRATCH 10 -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) || defined(Py_STACKREF_DEBUG) #define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ /* +1 because vectorcall might use -1 to write self */ \ PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ @@ -461,7 +461,7 @@ do { \ assert(NAME != NULL); #endif -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) || defined(Py_STACKREF_DEBUG) #define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ /* +1 because we +1 previously */ \ _PyObjectArray_Free(NAME - 1, NAME##_temp); @@ -470,7 +470,7 @@ do { \ (void)(NAME); #endif -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) || defined(Py_STACKREF_DEBUG) #define CONVERSION_FAILED(NAME) ((NAME) == NULL) #else #define CONVERSION_FAILED(NAME) (0) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6e752c57cd70f3..22335021faaa6d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -850,7 +850,7 @@ */ assert(Py_REFCNT(left_o) >= 2); PyStackRef_CLOSE(left); - PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ee5c55a832d460..bed16b60b76a2f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -208,7 +208,7 @@ */ assert(Py_REFCNT(left_o) >= 2); PyStackRef_CLOSE(left); - PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); @@ -1675,16 +1675,16 @@ callargs_st = tuple; func_st = func; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); PyObject *result_o; assert(!_PyErr_Occurred(tstate)); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-1 - (oparg & 1)] = callargs_st; @@ -1724,19 +1724,22 @@ if (Py_TYPE(func) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -3 - (oparg & 1); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; @@ -1745,6 +1748,10 @@ frame->return_offset = 1; DISPATCH_INLINED(new_frame); } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-1 - (oparg & 1)] = callargs_st; if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/pystate.c b/Python/pystate.c index 839413a65a42fb..c546b7c3a9f10e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -19,6 +19,7 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT +#include "pycore_stackref.h" // Py_STACKREF_DEBUG #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() #include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts() @@ -663,6 +664,23 @@ init_interpreter(PyInterpreterState *interp, /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); } +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + interp->next_stackref = 1; + _Py_hashtable_allocator_t alloc = { + .malloc = malloc, + .free = free, + }; + interp->stackref_debug_table = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, + NULL, + &alloc + ); + _Py_stackref_associate(interp, Py_None, PyStackRef_None); + _Py_stackref_associate(interp, Py_False, PyStackRef_False); + _Py_stackref_associate(interp, Py_True, PyStackRef_True); +#endif interp->_initialized = 1; return _PyStatus_OK(); @@ -768,6 +786,11 @@ PyInterpreterState_New(void) return interp; } +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) +extern void +_Py_stackref_report_leaks(PyInterpreterState *interp); +#endif + static void interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) { @@ -877,6 +900,12 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + _Py_stackref_report_leaks(interp); + _Py_hashtable_destroy(interp->stackref_debug_table); + interp->stackref_debug_table = NULL; +#endif + if (tstate->interp == interp) { /* We are now safe to fix tstate->_status.cleared. */ // XXX Do this (much) earlier? diff --git a/Python/stackrefs.c b/Python/stackrefs.c new file mode 100644 index 00000000000000..9bb46897685570 --- /dev/null +++ b/Python/stackrefs.c @@ -0,0 +1,156 @@ + +#include "Python.h" + +#include "pycore_stackref.h" + +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + +#if SIZEOF_VOID_P < 8 +#error "Py_STACKREF_DEBUG requires 64 bit machine" +#endif + +#include "pycore_interp.h" +#include "pycore_hashtable.h" + +typedef struct _table_entry { + PyObject *obj; + const char *classname; + const char *filename; + int linenumber; + const char *filename_borrow; + int linenumber_borrow; +} TableEntry; + +TableEntry * +make_table_entry(PyObject *obj, const char *filename, int linenumber) +{ + TableEntry *result = malloc(sizeof(TableEntry)); + if (result == NULL) { + return NULL; + } + result->obj = obj; + result->classname = Py_TYPE(obj)->tp_name; + result->filename = filename; + result->linenumber = linenumber; + result->filename_borrow = NULL; + return result; +} + +PyObject * +_Py_stackref_get_object(_PyStackRef ref) +{ + if (ref.index == 0) { + return NULL; + } + PyInterpreterState *interp = PyInterpreterState_Get(); + assert(interp != NULL); + if (ref.index >= interp->next_stackref) { + _Py_FatalErrorFormat(__func__, "Garbled stack ref with ID %" PRIu64 "\n", ref.index); + } + TableEntry *entry = _Py_hashtable_get(interp->stackref_debug_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, "Accessing closed stack ref with ID %" PRIu64 "\n", ref.index); + } + return entry->obj; +} + +PyObject * +_Py_stackref_close(_PyStackRef ref) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + if (ref.index >= interp->next_stackref) { + _Py_FatalErrorFormat(__func__, "Garbled stack ref with ID %" PRIu64 "\n", ref.index); + } + PyObject *obj; + if (ref.index <= LAST_PREDEFINED_STACKREF_INDEX) { + // Pre-allocated reference to None, False or True -- Do not clear + TableEntry *entry = _Py_hashtable_get(interp->stackref_debug_table, (void *)ref.index); + obj = entry->obj; + } + else { + TableEntry *entry = _Py_hashtable_steal(interp->stackref_debug_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 "\n", (void *)ref.index); + } + obj = entry->obj; + free(entry); + } + return obj; +} + +_PyStackRef +_Py_stackref_create(PyObject *obj, const char *filename, int linenumber) +{ + if (obj == NULL) { + Py_FatalError("Cannot create a stackref for NULL"); + } + PyInterpreterState *interp = PyInterpreterState_Get(); + uint64_t new_id = interp->next_stackref++; + TableEntry *entry = make_table_entry(obj, filename, linenumber); + if (entry == NULL) { + Py_FatalError("No memory left for stackref debug table"); + } + if (_Py_hashtable_set(interp->stackref_debug_table, (void *)new_id, entry) < 0) { + Py_FatalError("No memory left for stackref debug table"); + } + return (_PyStackRef){ .index = new_id }; +} + +void +_Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber) +{ + if (ref.index <= LAST_PREDEFINED_STACKREF_INDEX) { + return; + } + PyInterpreterState *interp = PyInterpreterState_Get(); + TableEntry *entry = _Py_hashtable_get(interp->stackref_debug_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 "\n", (void *)ref.index); + } + entry->filename_borrow = filename; + entry->linenumber_borrow = linenumber; +} + + +void +_Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref) +{ + assert(interp->next_stackref >= ref.index); + interp->next_stackref = ref.index+1; + TableEntry *entry = make_table_entry(obj, "builtin-object", 0); + if (entry == NULL) { + Py_FatalError("No memory left for stackref debug table"); + } + if (_Py_hashtable_set(interp->stackref_debug_table, (void *)ref.index, (void *)entry) < 0) { + Py_FatalError("No memory left for stackref debug table"); + } +} + + +static int +report_leak(_Py_hashtable_t *ht, const void *key, const void *value, void *leak) +{ + TableEntry *entry = (TableEntry *)value; + if (!_Py_IsStaticImmortal(entry->obj)) { + *(int *)leak = 1; + printf("Stackref leak. Refers to instance of %s at %p. Created at %s:%d", + entry->classname, entry->obj, entry->filename, entry->linenumber); + if (entry->filename_borrow != NULL) { + printf(". Last borrow at %s:%d",entry->filename_borrow, entry->linenumber_borrow); + } + printf("\n"); + } + return 0; +} + +void +_Py_stackref_report_leaks(PyInterpreterState *interp) +{ + int leak = 0; + _Py_hashtable_foreach(interp->stackref_debug_table, report_leak, &leak); + if (leak) { + Py_FatalError("Stackrefs leaked."); + } +} + +#endif From 5a584c8f54bbeceae7ffa501291e29b7ddc8a0b9 Mon Sep 17 00:00:00 2001 From: shallow-beach <96891913+shallow-beach@users.noreply.github.com> Date: Fri, 20 Dec 2024 09:09:56 -0800 Subject: [PATCH 261/427] Python Tutorial typo fix (#128077) --- Doc/tutorial/classes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 492568961d8a51..9d0fab8861d2a9 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -325,7 +325,7 @@ Now what can we do with instance objects? The only operations understood by instance objects are attribute references. There are two kinds of valid attribute names: data attributes and methods. -*data attributes* correspond to "instance variables" in Smalltalk, and to "data +*Data attributes* correspond to "instance variables" in Smalltalk, and to "data members" in C++. Data attributes need not be declared; like local variables, they spring into existence when they are first assigned to. For example, if ``x`` is the instance of :class:`!MyClass` created above, the following piece of From 3879ca0100942ae15a09ac22889cbe3e46d424eb Mon Sep 17 00:00:00 2001 From: Nico-Posada <102486290+Nico-Posada@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:20:31 -0600 Subject: [PATCH 262/427] gh-128049: Fix type confusion bug with the return value of a custom ExceptionGroup split function (#128079) --- Lib/test/test_except_star.py | 43 +++++++++++++++++++ ...-12-18-14-22-48.gh-issue-128079.SUD5le.rst | 5 +++ Python/ceval.c | 21 ++++++++- 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst diff --git a/Lib/test/test_except_star.py b/Lib/test/test_except_star.py index c49c6008e08e8c..284907f61213f8 100644 --- a/Lib/test/test_except_star.py +++ b/Lib/test/test_except_star.py @@ -952,6 +952,49 @@ def derive(self, excs): self.assertExceptionIsLike(tes, FalsyEG("eg", [TypeError(1)])) self.assertExceptionIsLike(ves, FalsyEG("eg", [ValueError(2)])) + def test_exception_group_subclass_with_bad_split_func(self): + # see gh-128049. + class BadEG1(ExceptionGroup): + def split(self, *args): + return "NOT A 2-TUPLE!" + + class BadEG2(ExceptionGroup): + def split(self, *args): + return ("NOT A 2-TUPLE!",) + + eg_list = [ + (BadEG1("eg", [OSError(123), ValueError(456)]), + r"split must return a tuple, not str"), + (BadEG2("eg", [OSError(123), ValueError(456)]), + r"split must return a 2-tuple, got tuple of size 1") + ] + + for eg_class, msg in eg_list: + with self.assertRaisesRegex(TypeError, msg) as m: + try: + raise eg_class + except* ValueError: + pass + except* OSError: + pass + + self.assertExceptionIsLike(m.exception.__context__, eg_class) + + # we allow tuples of length > 2 for backwards compatibility + class WeirdEG(ExceptionGroup): + def split(self, *args): + return super().split(*args) + ("anything", 123456, None) + + try: + raise WeirdEG("eg", [OSError(123), ValueError(456)]) + except* OSError as e: + oeg = e + except* ValueError as e: + veg = e + + self.assertExceptionIsLike(oeg, WeirdEG("eg", [OSError(123)])) + self.assertExceptionIsLike(veg, WeirdEG("eg", [ValueError(456)])) + class TestExceptStarCleanup(ExceptStarTest): def test_sys_exception_restored(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst new file mode 100644 index 00000000000000..8da4e677f068a3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-18-14-22-48.gh-issue-128079.SUD5le.rst @@ -0,0 +1,5 @@ +Fix a bug where :keyword:`except* ` does not properly check the +return value of an :exc:`ExceptionGroup`'s :meth:`~BaseExceptionGroup.split` +function, leading to a crash in some cases. Now when :meth:`~BaseExceptionGroup.split` +returns an invalid object, :keyword:`except* ` raises a :exc:`TypeError` +with the original raised :exc:`ExceptionGroup` object chained to it. diff --git a/Python/ceval.c b/Python/ceval.c index 3cf11b663c571b..e92a11b16cec81 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2134,8 +2134,25 @@ _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, if (pair == NULL) { return -1; } - assert(PyTuple_CheckExact(pair)); - assert(PyTuple_GET_SIZE(pair) == 2); + + if (!PyTuple_CheckExact(pair)) { + PyErr_Format(PyExc_TypeError, + "%.200s.split must return a tuple, not %.200s", + Py_TYPE(exc_value)->tp_name, Py_TYPE(pair)->tp_name); + Py_DECREF(pair); + return -1; + } + + // allow tuples of length > 2 for backwards compatibility + if (PyTuple_GET_SIZE(pair) < 2) { + PyErr_Format(PyExc_TypeError, + "%.200s.split must return a 2-tuple, " + "got tuple of size %zd", + Py_TYPE(exc_value)->tp_name, PyTuple_GET_SIZE(pair)); + Py_DECREF(pair); + return -1; + } + *match = Py_NewRef(PyTuple_GET_ITEM(pair, 0)); *rest = Py_NewRef(PyTuple_GET_ITEM(pair, 1)); Py_DECREF(pair); From 2a66dd33dfc0b845042da9bb54aaa4e890733f54 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 20 Dec 2024 20:40:58 +0100 Subject: [PATCH 263/427] gh-112328: Make EnumDict usable on its own and document it (GH-123669) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rafi Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) Co-authored-by: Ethan Furman --- Doc/library/enum.rst | 26 ++++++++++----- Doc/whatsnew/3.13.rst | 6 ++-- Lib/enum.py | 10 +++--- Lib/test/test_enum.py | 33 ++++++++++++++++++- ...-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst | 1 + 5 files changed, 60 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 51292a11f507c4..8ca949368db4ff 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -110,6 +110,10 @@ Module Contents ``KEEP`` which allows for more fine-grained control over how invalid values are dealt with in an enumeration. + :class:`EnumDict` + + A subclass of :class:`dict` for use when subclassing :class:`EnumType`. + :class:`auto` Instances are replaced with an appropriate value for Enum members. @@ -149,14 +153,10 @@ Module Contents Return a list of all power-of-two integers contained in a flag. - :class:`EnumDict` - - A subclass of :class:`dict` for use when subclassing :class:`EnumType`. - .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` .. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` -.. versionadded:: 3.14 ``EnumDict`` +.. versionadded:: 3.13 ``EnumDict`` --------------- @@ -830,13 +830,23 @@ Data Types .. class:: EnumDict - *EnumDict* is a subclass of :class:`dict` for use when subclassing :class:`EnumType`. + *EnumDict* is a subclass of :class:`dict` that is used as the namespace + for defining enum classes (see :ref:`prepare`). + It is exposed to allow subclasses of :class:`EnumType` with advanced + behavior like having multiple values per member. + It should be called with the name of the enum class being created, otherwise + private names and internal classes will not be handled correctly. + + Note that only the :class:`~collections.abc.MutableMapping` interface + (:meth:`~object.__setitem__` and :meth:`~dict.update`) is overridden. + It may be possible to bypass the checks using other :class:`!dict` + operations like :meth:`|= `. .. attribute:: EnumDict.member_names - Return list of member names. + A list of member names. - .. versionadded:: 3.14 + .. versionadded:: 3.13 --------------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a291122aefc2ce..c8e0f94f4246fb 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -879,11 +879,13 @@ email (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve the :cve:`2023-27043` fix.) + enum ---- -* :class:`~enum.EnumDict` has been made public in :mod:`enum` to better support - subclassing :class:`~enum.EnumType`. +* :class:`~enum.EnumDict` has been made public to better support subclassing + :class:`~enum.EnumType`. + fractions --------- diff --git a/Lib/enum.py b/Lib/enum.py index ccc1da42206474..04443471b40bff 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -342,12 +342,13 @@ class EnumDict(dict): EnumType will use the names found in self._member_names as the enumeration member names. """ - def __init__(self): + def __init__(self, cls_name=None): super().__init__() self._member_names = {} # use a dict -- faster look-up than a list, and keeps insertion order since 3.7 self._last_values = [] self._ignore = [] self._auto_called = False + self._cls_name = cls_name def __setitem__(self, key, value): """ @@ -358,7 +359,7 @@ def __setitem__(self, key, value): Single underscore (sunder) names are reserved. """ - if _is_private(self._cls_name, key): + if self._cls_name is not None and _is_private(self._cls_name, key): # do nothing, name will be a normal attribute pass elif _is_sunder(key): @@ -406,7 +407,7 @@ def __setitem__(self, key, value): value = value.value elif _is_descriptor(value): pass - elif _is_internal_class(self._cls_name, value): + elif self._cls_name is not None and _is_internal_class(self._cls_name, value): # do nothing, name will be a normal attribute pass else: @@ -478,8 +479,7 @@ def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist metacls._check_for_existing_members_(cls, bases) # create the namespace dict - enum_dict = EnumDict() - enum_dict._cls_name = cls + enum_dict = EnumDict(cls) # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(cls, bases) if first_enum is not None: diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index b9e13fb8c3585e..8884295b1ab89c 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -14,7 +14,7 @@ from enum import Enum, EnumMeta, IntEnum, StrEnum, EnumType, Flag, IntFlag, unique, auto from enum import STRICT, CONFORM, EJECT, KEEP, _simple_enum, _test_simple_enum from enum import verify, UNIQUE, CONTINUOUS, NAMED_FLAGS, ReprEnum -from enum import member, nonmember, _iter_bits_lsb +from enum import member, nonmember, _iter_bits_lsb, EnumDict from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL from test import support @@ -5440,6 +5440,37 @@ def test_convert_repr_and_str(self): self.assertEqual(format(test_type.CONVERT_STRING_TEST_NAME_A), '5') +class TestEnumDict(unittest.TestCase): + def test_enum_dict_in_metaclass(self): + """Test that EnumDict is usable as a class namespace""" + class Meta(type): + @classmethod + def __prepare__(metacls, cls, bases, **kwds): + return EnumDict(cls) + + class MyClass(metaclass=Meta): + a = 1 + + with self.assertRaises(TypeError): + a = 2 # duplicate + + with self.assertRaises(ValueError): + _a_sunder_ = 3 + + def test_enum_dict_standalone(self): + """Test that EnumDict is usable on its own""" + enumdict = EnumDict() + enumdict['a'] = 1 + + with self.assertRaises(TypeError): + enumdict['a'] = 'other value' + + # Only MutableMapping interface is overridden for now. + # If this stops passing, update the documentation. + enumdict |= {'a': 'other value'} + self.assertEqual(enumdict['a'], 'other value') + + # helpers def enum_dir(cls): diff --git a/Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst b/Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst new file mode 100644 index 00000000000000..96da94a9f211af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-04-14-13-14.gh-issue-121720.z9hhXQ.rst @@ -0,0 +1 @@ +:class:`enum.EnumDict` can now be used without resorting to private API. From a959ea1b0a026ff118975b9b539513b06dde3190 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 22 Dec 2024 01:17:59 +0000 Subject: [PATCH 264/427] GH-127807: pathlib ABCs: remove `PurePathBase._raw_paths` (#127883) Remove the `PurePathBase` initializer, and make `with_segments()` and `__str__()` abstract. This allows us to drop the `_raw_paths` attribute, and also the `Parser.join()` protocol method. --- Lib/pathlib/_abc.py | 40 +++-------- Lib/pathlib/_local.py | 37 ++++++++-- Lib/pathlib/_types.py | 1 - Lib/test/test_pathlib/test_pathlib.py | 25 +++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 85 ++++++++--------------- 5 files changed, 92 insertions(+), 96 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index b4560295300c28..4402efe3a02310 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -44,49 +44,25 @@ class PurePathBase: """Base class for pure path objects. This class *does not* provide several magic methods that are defined in - its subclass PurePath. They are: __fspath__, __bytes__, __reduce__, - __hash__, __eq__, __lt__, __le__, __gt__, __ge__. Its initializer and path - joining methods accept only strings, not os.PathLike objects more broadly. + its subclass PurePath. They are: __init__, __fspath__, __bytes__, + __reduce__, __hash__, __eq__, __lt__, __le__, __gt__, __ge__. """ - __slots__ = ( - # The `_raw_paths` slot stores unjoined string paths. This is set in - # the `__init__()` method. - '_raw_paths', - ) + __slots__ = () parser = posixpath _globber = PathGlobber - def __init__(self, *args): - for arg in args: - if not isinstance(arg, str): - raise TypeError( - f"argument should be a str, not {type(arg).__name__!r}") - self._raw_paths = list(args) - def with_segments(self, *pathsegments): """Construct a new path object from any number of path-like objects. Subclasses may override this method to customize how new path objects are created from methods like `iterdir()`. """ - return type(self)(*pathsegments) + raise NotImplementedError def __str__(self): """Return the string representation of the path, suitable for passing to system calls.""" - paths = self._raw_paths - if len(paths) == 1: - return paths[0] - elif paths: - # Join path segments from the initializer. - path = self.parser.join(*paths) - # Cache the joined path. - paths.clear() - paths.append(path) - return path - else: - paths.append('') - return '' + raise NotImplementedError def as_posix(self): """Return the string representation of the path with forward (/) @@ -234,17 +210,17 @@ def joinpath(self, *pathsegments): paths) or a totally different path (if one of the arguments is anchored). """ - return self.with_segments(*self._raw_paths, *pathsegments) + return self.with_segments(str(self), *pathsegments) def __truediv__(self, key): try: - return self.with_segments(*self._raw_paths, key) + return self.with_segments(str(self), key) except TypeError: return NotImplemented def __rtruediv__(self, key): try: - return self.with_segments(key, *self._raw_paths) + return self.with_segments(key, str(self)) except TypeError: return NotImplemented diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index b933dd512eeb28..7689c10604d4e6 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -77,6 +77,10 @@ class PurePath(PurePathBase): """ __slots__ = ( + # The `_raw_paths` slot stores unjoined string paths. This is set in + # the `__init__()` method. + '_raw_paths', + # The `_drv`, `_root` and `_tail_cached` slots store parsed and # normalized parts of the path. They are set when any of the `drive`, # `root` or `_tail` properties are accessed for the first time. The @@ -140,9 +144,15 @@ def __init__(self, *args): "object where __fspath__ returns a str, " f"not {type(path).__name__!r}") paths.append(path) - # Avoid calling super().__init__, as an optimisation self._raw_paths = paths + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + def joinpath(self, *pathsegments): """Combine this path with one or several arguments, and return a new path representing either a subpath (if all arguments are relative @@ -304,14 +314,29 @@ def _parse_pattern(cls, pattern): parts.append('') return parts + @property + def _raw_path(self): + paths = self._raw_paths + if len(paths) == 1: + return paths[0] + elif paths: + # Join path segments from the initializer. + path = self.parser.join(*paths) + # Cache the joined path. + paths.clear() + paths.append(path) + return path + else: + paths.append('') + return '' + @property def drive(self): """The drive prefix (letter or UNC path), if any.""" try: return self._drv except AttributeError: - raw_path = PurePathBase.__str__(self) - self._drv, self._root, self._tail_cached = self._parse_path(raw_path) + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) return self._drv @property @@ -320,8 +345,7 @@ def root(self): try: return self._root except AttributeError: - raw_path = PurePathBase.__str__(self) - self._drv, self._root, self._tail_cached = self._parse_path(raw_path) + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) return self._root @property @@ -329,8 +353,7 @@ def _tail(self): try: return self._tail_cached except AttributeError: - raw_path = PurePathBase.__str__(self) - self._drv, self._root, self._tail_cached = self._parse_path(raw_path) + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) return self._tail_cached @property diff --git a/Lib/pathlib/_types.py b/Lib/pathlib/_types.py index 60df94d0b46049..baa4a7e2af5b68 100644 --- a/Lib/pathlib/_types.py +++ b/Lib/pathlib/_types.py @@ -14,7 +14,6 @@ class Parser(Protocol): """ sep: str - def join(self, path: str, *paths: str) -> str: ... def split(self, path: str) -> tuple[str, str]: ... def splitdrive(self, path: str) -> tuple[str, str]: ... def splitext(self, path: str) -> tuple[str, str]: ... diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index ac3a3b4f15c10e..ef482c311542fa 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -229,6 +229,31 @@ def test_fspath_common(self): self._check_str(p.__fspath__(), ('a/b',)) self._check_str(os.fspath(p), ('a/b',)) + def test_bytes(self): + P = self.cls + with self.assertRaises(TypeError): + P(b'a') + with self.assertRaises(TypeError): + P(b'a', 'b') + with self.assertRaises(TypeError): + P('a', b'b') + with self.assertRaises(TypeError): + P('a').joinpath(b'b') + with self.assertRaises(TypeError): + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') + def test_bytes_exc_message(self): P = self.cls message = (r"argument should be a str or an os\.PathLike object " diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index e230dd188799a5..9198a0cbc45cee 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -53,7 +53,15 @@ def test_parser(self): class DummyPurePath(PurePathBase): - __slots__ = () + __slots__ = ('_segments',) + + def __init__(self, *segments): + self._segments = segments + + def __str__(self): + if self._segments: + return self.parser.join(*self._segments) + return '' def __eq__(self, other): if not isinstance(other, DummyPurePath): @@ -66,6 +74,9 @@ def __hash__(self): def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + def with_segments(self, *pathsegments): + return type(self)(*pathsegments) + class DummyPurePathTest(unittest.TestCase): cls = DummyPurePath @@ -97,30 +108,11 @@ def test_constructor_common(self): P('a/b/c') P('/a/b/c') - def test_bytes(self): - P = self.cls - with self.assertRaises(TypeError): - P(b'a') - with self.assertRaises(TypeError): - P(b'a', 'b') - with self.assertRaises(TypeError): - P('a', b'b') - with self.assertRaises(TypeError): - P('a').joinpath(b'b') - with self.assertRaises(TypeError): - P('a') / b'b' - with self.assertRaises(TypeError): - b'a' / P('b') - with self.assertRaises(TypeError): - P('a').match(b'b') - with self.assertRaises(TypeError): - P('a').relative_to(b'b') - with self.assertRaises(TypeError): - P('a').with_name(b'b') - with self.assertRaises(TypeError): - P('a').with_stem(b'b') - with self.assertRaises(TypeError): - P('a').with_suffix(b'b') + def test_fspath_common(self): + self.assertRaises(TypeError, os.fspath, self.cls('')) + + def test_as_bytes_common(self): + self.assertRaises(TypeError, bytes, self.cls('')) def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object @@ -1286,36 +1278,6 @@ def test_is_absolute_windows(self): # Tests for the virtual classes. # -class PathBaseTest(PurePathBaseTest): - cls = PathBase - - def test_not_implemented_error(self): - p = self.cls('') - e = NotImplementedError - self.assertRaises(e, p.stat) - self.assertRaises(e, p.exists) - self.assertRaises(e, p.is_dir) - self.assertRaises(e, p.is_file) - self.assertRaises(e, p.is_symlink) - self.assertRaises(e, p.open) - self.assertRaises(e, p.read_bytes) - self.assertRaises(e, p.read_text) - self.assertRaises(e, p.write_bytes, b'foo') - self.assertRaises(e, p.write_text, 'foo') - self.assertRaises(e, p.iterdir) - self.assertRaises(e, lambda: list(p.glob('*'))) - self.assertRaises(e, lambda: list(p.rglob('*'))) - self.assertRaises(e, lambda: list(p.walk())) - self.assertRaises(e, p.readlink) - self.assertRaises(e, p.symlink_to, 'foo') - self.assertRaises(e, p.mkdir) - - def test_fspath_common(self): - self.assertRaises(TypeError, os.fspath, self.cls('')) - - def test_as_bytes_common(self): - self.assertRaises(TypeError, bytes, self.cls('')) - class DummyPathIO(io.BytesIO): """ @@ -1342,11 +1304,19 @@ class DummyPath(PathBase): Simple implementation of PathBase that keeps files and directories in memory. """ - __slots__ = () + __slots__ = ('_segments') _files = {} _directories = {} + def __init__(self, *segments): + self._segments = segments + + def __str__(self): + if self._segments: + return self.parser.join(*self._segments) + return '' + def __eq__(self, other): if not isinstance(other, DummyPath): return NotImplemented @@ -1358,6 +1328,9 @@ def __hash__(self): def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + def with_segments(self, *pathsegments): + return type(self)(*pathsegments) + def stat(self, *, follow_symlinks=True): path = str(self).rstrip('/') if path in self._files: From f5ba74b81979b621e38be70ec3ddad1e7f1365ce Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 22 Dec 2024 01:41:38 +0000 Subject: [PATCH 265/427] GH-127807: pathlib ABCs: remove a few private attributes (#127851) From `PurePathBase` delete `_globber`, `_stack` and `_pattern_str`, and from `PathBase` delete `_glob_selector`. This helps avoid an unpleasant surprise for a users who try to use these names. --- Lib/pathlib/_abc.py | 86 +++++++++++++++++++------------------------ Lib/pathlib/_local.py | 34 +++++++++++++---- 2 files changed, 64 insertions(+), 56 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 4402efe3a02310..b521c757561a99 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -25,6 +25,23 @@ def _is_case_sensitive(parser): return parser.normcase('Aa') == 'Aa' +def _explode_path(path): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + split = path.parser.split + path = str(path) + parent, name = split(path) + names = [] + while path != parent: + names.append(name) + path = parent + parent, name = split(path) + return path, names + + class PathGlobber(_GlobberBase): """ Class providing shell-style globbing for path objects. @@ -50,7 +67,6 @@ class PurePathBase: __slots__ = () parser = posixpath - _globber = PathGlobber def with_segments(self, *pathsegments): """Construct a new path object from any number of path-like objects. @@ -82,7 +98,7 @@ def root(self): @property def anchor(self): """The concatenation of the drive and root, or ''.""" - return self._stack[0] + return _explode_path(self)[0] @property def name(self): @@ -160,8 +176,8 @@ def relative_to(self, other, *, walk_up=False): """ if not isinstance(other, PurePathBase): other = self.with_segments(other) - anchor0, parts0 = self._stack - anchor1, parts1 = other._stack + anchor0, parts0 = _explode_path(self) + anchor1, parts1 = _explode_path(other) if anchor0 != anchor1: raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") while parts0 and parts1 and parts0[-1] == parts1[-1]: @@ -183,8 +199,8 @@ def is_relative_to(self, other): """ if not isinstance(other, PurePathBase): other = self.with_segments(other) - anchor0, parts0 = self._stack - anchor1, parts1 = other._stack + anchor0, parts0 = _explode_path(self) + anchor1, parts1 = _explode_path(other) if anchor0 != anchor1: return False while parts0 and parts1 and parts0[-1] == parts1[-1]: @@ -199,7 +215,7 @@ def is_relative_to(self, other): def parts(self): """An object providing sequence-like access to the components in the filesystem path.""" - anchor, parts = self._stack + anchor, parts = _explode_path(self) if anchor: parts.append(anchor) return tuple(reversed(parts)) @@ -224,23 +240,6 @@ def __rtruediv__(self, key): except TypeError: return NotImplemented - @property - def _stack(self): - """ - Split the path into a 2-tuple (anchor, parts), where *anchor* is the - uppermost parent of the path (equivalent to path.parents[-1]), and - *parts* is a reversed list of parts following the anchor. - """ - split = self.parser.split - path = str(self) - parent, name = split(path) - names = [] - while path != parent: - names.append(name) - path = parent - parent, name = split(path) - return path, names - @property def parent(self): """The logical parent of the path.""" @@ -268,11 +267,6 @@ def is_absolute(self): a drive).""" return self.parser.isabs(str(self)) - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" - return str(self) - def match(self, path_pattern, *, case_sensitive=None): """ Return True if this path matches the given pattern. If the pattern is @@ -293,7 +287,7 @@ def match(self, path_pattern, *, case_sensitive=None): return False if len(path_parts) > len(pattern_parts) and path_pattern.anchor: return False - globber = self._globber(sep, case_sensitive) + globber = PathGlobber(sep, case_sensitive) for path_part, pattern_part in zip(path_parts, pattern_parts): match = globber.compile(pattern_part) if match(path_part) is None: @@ -309,9 +303,9 @@ def full_match(self, pattern, *, case_sensitive=None): pattern = self.with_segments(pattern) if case_sensitive is None: case_sensitive = _is_case_sensitive(self.parser) - globber = self._globber(pattern.parser.sep, case_sensitive, recursive=True) - match = globber.compile(pattern._pattern_str) - return match(self._pattern_str) is not None + globber = PathGlobber(pattern.parser.sep, case_sensitive, recursive=True) + match = globber.compile(str(pattern)) + return match(str(self)) is not None @@ -463,29 +457,25 @@ def iterdir(self): """ raise NotImplementedError - def _glob_selector(self, parts, case_sensitive, recurse_symlinks): - if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.parser) - case_pedantic = False - else: - # The user has expressed a case sensitivity choice, but we don't - # know the case sensitivity of the underlying filesystem, so we - # must use scandir() for everything, including non-wildcard parts. - case_pedantic = True - recursive = True if recurse_symlinks else _no_recurse_symlinks - globber = self._globber(self.parser.sep, case_sensitive, case_pedantic, recursive) - return globber.selector(parts) - def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ if not isinstance(pattern, PurePathBase): pattern = self.with_segments(pattern) - anchor, parts = pattern._stack + anchor, parts = _explode_path(pattern) if anchor: raise NotImplementedError("Non-relative patterns are unsupported") - select = self._glob_selector(parts, case_sensitive, recurse_symlinks) + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.parser) + case_pedantic = False + elif case_sensitive == _is_case_sensitive(self.parser): + case_pedantic = False + else: + case_pedantic = True + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = PathGlobber(self.parser.sep, case_sensitive, case_pedantic, recursive) + select = globber.selector(parts) return select(self) def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 7689c10604d4e6..4897149d7e8a8e 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -5,7 +5,7 @@ import posixpath import sys from errno import EINVAL, EXDEV -from glob import _StringGlobber +from glob import _StringGlobber, _no_recurse_symlinks from itertools import chain from stat import S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from _collections_abc import Sequence @@ -112,7 +112,6 @@ class PurePath(PurePathBase): '_hash', ) parser = os.path - _globber = _StringGlobber def __new__(cls, *args, **kwargs): """Construct a PurePath from one or several strings and or existing @@ -513,13 +512,22 @@ def as_uri(self): from urllib.parse import quote_from_bytes return prefix + quote_from_bytes(os.fsencode(path)) - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" + def full_match(self, pattern, *, case_sensitive=None): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + # The string representation of an empty path is a single dot ('.'). Empty # paths shouldn't match wildcards, so we change it to the empty string. - path_str = str(self) - return '' if path_str == '.' else path_str + path = str(self) if self.parts else '' + pattern = str(pattern) if pattern.parts else '' + globber = _StringGlobber(self.parser.sep, case_sensitive, recursive=True) + return globber.compile(pattern)(path) is not None # Subclassing os.PathLike makes isinstance() checks slower, # which in turn makes Path construction slower. Register instead! @@ -749,8 +757,18 @@ def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): kind, including directories) matching the given relative pattern. """ sys.audit("pathlib.Path.glob", self, pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + case_pedantic = False + else: + # The user has expressed a case sensitivity choice, but we don't + # know the case sensitivity of the underlying filesystem, so we + # must use scandir() for everything, including non-wildcard parts. + case_pedantic = True parts = self._parse_pattern(pattern) - select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = _StringGlobber(self.parser.sep, case_sensitive, case_pedantic, recursive) + select = globber.selector(parts[::-1]) root = str(self) paths = select(root) From 8d9f52a7be5c09c0fd4423943edadaacf6d7f917 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 22 Dec 2024 02:22:08 +0000 Subject: [PATCH 266/427] GH-127807: pathlib ABCs: move private copying methods to dedicated class (#127810) Move 9 private `PathBase` attributes and methods into a new `CopyWorker` class. Change `PathBase.copy` from a method to a `CopyWorker` instance. The methods remain private in the `CopyWorker` class. In future we might make some/all of them public so that user subclasses of `PathBase` can customize the copying process (in particular reading/writing of metadata,) but we'd need to make `PathBase` public first. --- Lib/pathlib/_abc.py | 228 +++++++++++++++++++++++------------------- Lib/pathlib/_local.py | 183 ++++++++++++++++++++++++--------- Lib/pathlib/_os.py | 98 ------------------ 3 files changed, 261 insertions(+), 248 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index b521c757561a99..6acc29ebab2bc5 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -57,6 +57,132 @@ def concat_path(path, text): return path.with_segments(str(path) + text) +class CopyWorker: + """ + Class that implements copying between path objects. An instance of this + class is available from the PathBase.copy property; it's made callable so + that PathBase.copy() can be treated as a method. + + The target path's CopyWorker drives the process from its _create() method. + Files and directories are exchanged by calling methods on the source and + target paths, and metadata is exchanged by calling + source.copy._read_metadata() and target.copy._write_metadata(). + """ + __slots__ = ('_path',) + + def __init__(self, path): + self._path = path + + def __call__(self, target, follow_symlinks=True, dirs_exist_ok=False, + preserve_metadata=False): + """ + Recursively copy this file or directory tree to the given destination. + """ + if not isinstance(target, PathBase): + target = self._path.with_segments(target) + + # Delegate to the target path's CopyWorker object. + return target.copy._create(self._path, follow_symlinks, dirs_exist_ok, preserve_metadata) + + _readable_metakeys = frozenset() + + def _read_metadata(self, metakeys, *, follow_symlinks=True): + """ + Returns path metadata as a dict with string keys. + """ + raise NotImplementedError + + _writable_metakeys = frozenset() + + def _write_metadata(self, metadata, *, follow_symlinks=True): + """ + Sets path metadata from the given dict with string keys. + """ + raise NotImplementedError + + def _create(self, source, follow_symlinks, dirs_exist_ok, preserve_metadata): + self._ensure_distinct_path(source) + if preserve_metadata: + metakeys = self._writable_metakeys & source.copy._readable_metakeys + else: + metakeys = None + if not follow_symlinks and source.is_symlink(): + self._create_symlink(source, metakeys) + elif source.is_dir(): + self._create_dir(source, metakeys, follow_symlinks, dirs_exist_ok) + else: + self._create_file(source, metakeys) + return self._path + + def _create_dir(self, source, metakeys, follow_symlinks, dirs_exist_ok): + """Copy the given directory to our path.""" + children = list(source.iterdir()) + self._path.mkdir(exist_ok=dirs_exist_ok) + for src in children: + dst = self._path.joinpath(src.name) + if not follow_symlinks and src.is_symlink(): + dst.copy._create_symlink(src, metakeys) + elif src.is_dir(): + dst.copy._create_dir(src, metakeys, follow_symlinks, dirs_exist_ok) + else: + dst.copy._create_file(src, metakeys) + if metakeys: + metadata = source.copy._read_metadata(metakeys) + if metadata: + self._write_metadata(metadata) + + def _create_file(self, source, metakeys): + """Copy the given file to our path.""" + self._ensure_different_file(source) + with source.open('rb') as source_f: + try: + with self._path.open('wb') as target_f: + copyfileobj(source_f, target_f) + except IsADirectoryError as e: + if not self._path.exists(): + # Raise a less confusing exception. + raise FileNotFoundError( + f'Directory does not exist: {self._path}') from e + raise + if metakeys: + metadata = source.copy._read_metadata(metakeys) + if metadata: + self._write_metadata(metadata) + + def _create_symlink(self, source, metakeys): + """Copy the given symbolic link to our path.""" + self._path.symlink_to(source.readlink()) + if metakeys: + metadata = source.copy._read_metadata(metakeys, follow_symlinks=False) + if metadata: + self._write_metadata(metadata, follow_symlinks=False) + + def _ensure_different_file(self, source): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + pass + + def _ensure_distinct_path(self, source): + """ + Raise OSError(EINVAL) if the other path is within this path. + """ + # Note: there is no straightforward, foolproof algorithm to determine + # if one directory is within another (a particularly perverse example + # would be a single network share mounted in one location via NFS, and + # in another location via CIFS), so we simply checks whether the + # other path is lexically equal to, or within, this path. + if source == self._path: + err = OSError(EINVAL, "Source and target are the same path") + elif source in self._path.parents: + err = OSError(EINVAL, "Source path is a parent of target path") + else: + return + err.filename = str(source) + err.filename2 = str(self._path) + raise err + + class PurePathBase: """Base class for pure path objects. @@ -374,31 +500,6 @@ def is_symlink(self): except (OSError, ValueError): return False - def _ensure_different_file(self, other_path): - """ - Raise OSError(EINVAL) if both paths refer to the same file. - """ - pass - - def _ensure_distinct_path(self, other_path): - """ - Raise OSError(EINVAL) if the other path is within this path. - """ - # Note: there is no straightforward, foolproof algorithm to determine - # if one directory is within another (a particularly perverse example - # would be a single network share mounted in one location via NFS, and - # in another location via CIFS), so we simply checks whether the - # other path is lexically equal to, or within, this path. - if self == other_path: - err = OSError(EINVAL, "Source and target are the same path") - elif self in other_path.parents: - err = OSError(EINVAL, "Source path is a parent of target path") - else: - return - err.filename = str(self) - err.filename2 = str(other_path) - raise err - def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ @@ -537,88 +638,13 @@ def symlink_to(self, target, target_is_directory=False): """ raise NotImplementedError - def _symlink_to_target_of(self, link): - """ - Make this path a symlink with the same target as the given link. This - is used by copy(). - """ - self.symlink_to(link.readlink()) - def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ Create a new directory at this given path. """ raise NotImplementedError - # Metadata keys supported by this path type. - _readable_metadata = _writable_metadata = frozenset() - - def _read_metadata(self, keys=None, *, follow_symlinks=True): - """ - Returns path metadata as a dict with string keys. - """ - raise NotImplementedError - - def _write_metadata(self, metadata, *, follow_symlinks=True): - """ - Sets path metadata from the given dict with string keys. - """ - raise NotImplementedError - - def _copy_metadata(self, target, *, follow_symlinks=True): - """ - Copies metadata (permissions, timestamps, etc) from this path to target. - """ - # Metadata types supported by both source and target. - keys = self._readable_metadata & target._writable_metadata - if keys: - metadata = self._read_metadata(keys, follow_symlinks=follow_symlinks) - target._write_metadata(metadata, follow_symlinks=follow_symlinks) - - def _copy_file(self, target): - """ - Copy the contents of this file to the given target. - """ - self._ensure_different_file(target) - with self.open('rb') as source_f: - try: - with target.open('wb') as target_f: - copyfileobj(source_f, target_f) - except IsADirectoryError as e: - if not target.exists(): - # Raise a less confusing exception. - raise FileNotFoundError( - f'Directory does not exist: {target}') from e - else: - raise - - def copy(self, target, *, follow_symlinks=True, dirs_exist_ok=False, - preserve_metadata=False): - """ - Recursively copy this file or directory tree to the given destination. - """ - if not isinstance(target, PathBase): - target = self.with_segments(target) - self._ensure_distinct_path(target) - stack = [(self, target)] - while stack: - src, dst = stack.pop() - if not follow_symlinks and src.is_symlink(): - dst._symlink_to_target_of(src) - if preserve_metadata: - src._copy_metadata(dst, follow_symlinks=False) - elif src.is_dir(): - children = src.iterdir() - dst.mkdir(exist_ok=dirs_exist_ok) - stack.extend((child, dst.joinpath(child.name)) - for child in children) - if preserve_metadata: - src._copy_metadata(dst) - else: - src._copy_file(dst) - if preserve_metadata: - src._copy_metadata(dst) - return target + copy = property(CopyWorker, doc=CopyWorker.__call__.__doc__) def copy_into(self, target_dir, *, follow_symlinks=True, dirs_exist_ok=False, preserve_metadata=False): diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 4897149d7e8a8e..915402e6c65b29 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -4,10 +4,10 @@ import os import posixpath import sys -from errno import EINVAL, EXDEV +from errno import * from glob import _StringGlobber, _no_recurse_symlinks from itertools import chain -from stat import S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from stat import S_IMODE, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from _collections_abc import Sequence try: @@ -19,9 +19,8 @@ except ImportError: grp = None -from pathlib._os import (copyfile, file_metadata_keys, read_file_metadata, - write_file_metadata) -from pathlib._abc import PurePathBase, PathBase +from pathlib._os import copyfile +from pathlib._abc import CopyWorker, PurePathBase, PathBase __all__ = [ @@ -66,6 +65,131 @@ def __repr__(self): return "<{}.parents>".format(type(self._path).__name__) +class _LocalCopyWorker(CopyWorker): + """This object implements the Path.copy callable. Don't try to construct + it yourself.""" + __slots__ = () + + _readable_metakeys = {'mode', 'times_ns'} + if hasattr(os.stat_result, 'st_flags'): + _readable_metakeys.add('flags') + if hasattr(os, 'listxattr'): + _readable_metakeys.add('xattrs') + _readable_metakeys = _writable_metakeys = frozenset(_readable_metakeys) + + def _read_metadata(self, metakeys, *, follow_symlinks=True): + metadata = {} + if 'mode' in metakeys or 'times_ns' in metakeys or 'flags' in metakeys: + st = self._path.stat(follow_symlinks=follow_symlinks) + if 'mode' in metakeys: + metadata['mode'] = S_IMODE(st.st_mode) + if 'times_ns' in metakeys: + metadata['times_ns'] = st.st_atime_ns, st.st_mtime_ns + if 'flags' in metakeys: + metadata['flags'] = st.st_flags + if 'xattrs' in metakeys: + try: + metadata['xattrs'] = [ + (attr, os.getxattr(self._path, attr, follow_symlinks=follow_symlinks)) + for attr in os.listxattr(self._path, follow_symlinks=follow_symlinks)] + except OSError as err: + if err.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + return metadata + + def _write_metadata(self, metadata, *, follow_symlinks=True): + def _nop(*args, ns=None, follow_symlinks=None): + pass + + if follow_symlinks: + # use the real function if it exists + def lookup(name): + return getattr(os, name, _nop) + else: + # use the real function only if it exists + # *and* it supports follow_symlinks + def lookup(name): + fn = getattr(os, name, _nop) + if fn in os.supports_follow_symlinks: + return fn + return _nop + + times_ns = metadata.get('times_ns') + if times_ns is not None: + lookup("utime")(self._path, ns=times_ns, follow_symlinks=follow_symlinks) + # We must copy extended attributes before the file is (potentially) + # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. + xattrs = metadata.get('xattrs') + if xattrs is not None: + for attr, value in xattrs: + try: + os.setxattr(self._path, attr, value, follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + mode = metadata.get('mode') + if mode is not None: + try: + lookup("chmod")(self._path, mode, follow_symlinks=follow_symlinks) + except NotImplementedError: + # if we got a NotImplementedError, it's because + # * follow_symlinks=False, + # * lchown() is unavailable, and + # * either + # * fchownat() is unavailable or + # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. + # (it returned ENOSUP.) + # therefore we're out of options--we simply cannot chown the + # symlink. give up, suppress the error. + # (which is what shutil always did in this circumstance.) + pass + flags = metadata.get('flags') + if flags is not None: + try: + lookup("chflags")(self._path, flags, follow_symlinks=follow_symlinks) + except OSError as why: + if why.errno not in (EOPNOTSUPP, ENOTSUP): + raise + + if copyfile: + # Use fast OS routine for local file copying where available. + def _create_file(self, source, metakeys): + """Copy the given file to the given target.""" + try: + source = os.fspath(source) + except TypeError: + if not isinstance(source, PathBase): + raise + super()._create_file(source, metakeys) + else: + copyfile(source, os.fspath(self._path)) + + if os.name == 'nt': + # Windows: symlink target might not exist yet if we're copying several + # files, so ensure we pass is_dir to os.symlink(). + def _create_symlink(self, source, metakeys): + """Copy the given symlink to the given target.""" + self._path.symlink_to(source.readlink(), source.is_dir()) + if metakeys: + metadata = source.copy._read_metadata(metakeys, follow_symlinks=False) + if metadata: + self._write_metadata(metadata, follow_symlinks=False) + + def _ensure_different_file(self, source): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + try: + if not self._path.samefile(source): + return + except (OSError, ValueError): + return + err = OSError(EINVAL, "Source and target are the same file") + err.filename = str(source) + err.filename2 = str(self._path) + raise err + + class PurePath(PurePathBase): """Base class for manipulating paths without I/O. @@ -678,20 +802,6 @@ def samefile(self, other_path): return (st.st_ino == other_st.st_ino and st.st_dev == other_st.st_dev) - def _ensure_different_file(self, other_path): - """ - Raise OSError(EINVAL) if both paths refer to the same file. - """ - try: - if not self.samefile(other_path): - return - except (OSError, ValueError): - return - err = OSError(EINVAL, "Source and target are the same file") - err.filename = str(self) - err.filename2 = str(other_path) - raise err - def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ @@ -932,24 +1042,6 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): if not exist_ok or not self.is_dir(): raise - _readable_metadata = _writable_metadata = file_metadata_keys - _read_metadata = read_file_metadata - _write_metadata = write_file_metadata - - if copyfile: - def _copy_file(self, target): - """ - Copy the contents of this file to the given target. - """ - try: - target = os.fspath(target) - except TypeError: - if not isinstance(target, PathBase): - raise - PathBase._copy_file(self, target) - else: - copyfile(os.fspath(self), target) - def chmod(self, mode, *, follow_symlinks=True): """ Change the permissions of the path, like os.chmod(). @@ -1019,16 +1111,17 @@ def replace(self, target): os.replace(self, target) return self.with_segments(target) + copy = property(_LocalCopyWorker, doc=_LocalCopyWorker.__call__.__doc__) + def move(self, target): """ Recursively move this file or directory tree to the given destination. """ - self._ensure_different_file(target) + if not isinstance(target, PathBase): + target = self.with_segments(target) + target.copy._ensure_different_file(self) try: return self.replace(target) - except TypeError: - if not isinstance(target, PathBase): - raise except OSError as err: if err.errno != EXDEV: raise @@ -1051,14 +1144,6 @@ def symlink_to(self, target, target_is_directory=False): f = f"{type(self).__name__}.symlink_to()" raise UnsupportedOperation(f"{f} is unsupported on this system") - if os.name == 'nt': - def _symlink_to_target_of(self, link): - """ - Make this path a symlink with the same target as the given link. - This is used by copy(). - """ - self.symlink_to(link.readlink(), link.is_dir()) - if hasattr(os, "link"): def hardlink_to(self, target): """ diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py index 642b3a57c59a1d..57bcaf3d680138 100644 --- a/Lib/pathlib/_os.py +++ b/Lib/pathlib/_os.py @@ -4,7 +4,6 @@ from errno import * import os -import stat import sys try: import fcntl @@ -163,100 +162,3 @@ def copyfileobj(source_f, target_f): write_target = target_f.write while buf := read_source(1024 * 1024): write_target(buf) - - -# Kinds of metadata supported by the operating system. -file_metadata_keys = {'mode', 'times_ns'} -if hasattr(os.stat_result, 'st_flags'): - file_metadata_keys.add('flags') -if hasattr(os, 'listxattr'): - file_metadata_keys.add('xattrs') -file_metadata_keys = frozenset(file_metadata_keys) - - -def read_file_metadata(path, keys=None, *, follow_symlinks=True): - """ - Returns local path metadata as a dict with string keys. - """ - if keys is None: - keys = file_metadata_keys - assert keys.issubset(file_metadata_keys) - result = {} - for key in keys: - if key == 'xattrs': - try: - result['xattrs'] = [ - (attr, os.getxattr(path, attr, follow_symlinks=follow_symlinks)) - for attr in os.listxattr(path, follow_symlinks=follow_symlinks)] - except OSError as err: - if err.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): - raise - continue - st = os.stat(path, follow_symlinks=follow_symlinks) - if key == 'mode': - result['mode'] = stat.S_IMODE(st.st_mode) - elif key == 'times_ns': - result['times_ns'] = st.st_atime_ns, st.st_mtime_ns - elif key == 'flags': - result['flags'] = st.st_flags - return result - - -def write_file_metadata(path, metadata, *, follow_symlinks=True): - """ - Sets local path metadata from the given dict with string keys. - """ - assert frozenset(metadata.keys()).issubset(file_metadata_keys) - - def _nop(*args, ns=None, follow_symlinks=None): - pass - - if follow_symlinks: - # use the real function if it exists - def lookup(name): - return getattr(os, name, _nop) - else: - # use the real function only if it exists - # *and* it supports follow_symlinks - def lookup(name): - fn = getattr(os, name, _nop) - if fn in os.supports_follow_symlinks: - return fn - return _nop - - times_ns = metadata.get('times_ns') - if times_ns is not None: - lookup("utime")(path, ns=times_ns, follow_symlinks=follow_symlinks) - # We must copy extended attributes before the file is (potentially) - # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. - xattrs = metadata.get('xattrs') - if xattrs is not None: - for attr, value in xattrs: - try: - os.setxattr(path, attr, value, follow_symlinks=follow_symlinks) - except OSError as e: - if e.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): - raise - mode = metadata.get('mode') - if mode is not None: - try: - lookup("chmod")(path, mode, follow_symlinks=follow_symlinks) - except NotImplementedError: - # if we got a NotImplementedError, it's because - # * follow_symlinks=False, - # * lchown() is unavailable, and - # * either - # * fchownat() is unavailable or - # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. - # (it returned ENOSUP.) - # therefore we're out of options--we simply cannot chown the - # symlink. give up, suppress the error. - # (which is what shutil always did in this circumstance.) - pass - flags = metadata.get('flags') - if flags is not None: - try: - lookup("chflags")(path, flags, follow_symlinks=follow_symlinks) - except OSError as why: - if why.errno not in (EOPNOTSUPP, ENOTSUP): - raise From 228f275737615cc9be713a8c3f9325b359bf8aec Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sun, 22 Dec 2024 01:47:41 -0800 Subject: [PATCH 267/427] gh-126664: revert: Use `else` instead of `finally` in docs explaining "with" (#128169) Revert "gh-126664: Use `else` instead of `finally` in "The with statement" documentation. (GH-126665)" This reverts commit 25257d61cfccc3b4189f96390a5c4db73fd5302c. --- Doc/reference/compound_stmts.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index e73ce44270b082..1b1e9f479cbe08 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -534,15 +534,18 @@ is semantically equivalent to:: enter = type(manager).__enter__ exit = type(manager).__exit__ value = enter(manager) + hit_except = False try: TARGET = value SUITE except: + hit_except = True if not exit(manager, *sys.exc_info()): raise - else: - exit(manager, None, None, None) + finally: + if not hit_except: + exit(manager, None, None, None) With more than one item, the context managers are processed as if multiple :keyword:`with` statements were nested:: From b66a4ad9fc32b63da2ba10db24cbc8f4e29f781a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 22 Dec 2024 12:46:02 +0000 Subject: [PATCH 268/427] gh-127949: fix resource warnings in `test_tasks.py` (#128172) --- Lib/test/test_asyncio/test_tasks.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 5b8979a8bbd13a..7d6d0564a9a9db 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2698,17 +2698,17 @@ def __str__(self): initial_refcount = sys.getrefcount(obj) coro = coroutine_function() - loop = asyncio.new_event_loop() - task = asyncio.Task.__new__(asyncio.Task) + with contextlib.closing(asyncio.EventLoop()) as loop: + task = asyncio.Task.__new__(asyncio.Task) - for _ in range(5): - with self.assertRaisesRegex(RuntimeError, 'break'): - task.__init__(coro, loop=loop, context=obj, name=Break()) + for _ in range(5): + with self.assertRaisesRegex(RuntimeError, 'break'): + task.__init__(coro, loop=loop, context=obj, name=Break()) - coro.close() - del task + coro.close() + del task - self.assertEqual(sys.getrefcount(obj), initial_refcount) + self.assertEqual(sys.getrefcount(obj), initial_refcount) def add_subclass_tests(cls): From f420bdd29fbc1a97ad20d88075c38c937c1f8479 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Sun, 22 Dec 2024 17:34:16 +0100 Subject: [PATCH 269/427] gh-119786: Fix typos in `InternalDocs/interpreter.md` (#128174) --- InternalDocs/interpreter.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index fa4a54fdc54fac..52702792c6cb7b 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -20,7 +20,7 @@ When the interpreter's [`PyEval_EvalCode()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) function is called to execute a `CodeObject`, it constructs a [`Frame`](frames.md) and calls [`_PyEval_EvalFrame()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) -to execute the code object in this frame. The frame hold the dynamic state of the +to execute the code object in this frame. The frame holds the dynamic state of the `CodeObject`'s execution, including the instruction pointer, the globals and builtins. It also has a reference to the `CodeObject` itself. @@ -153,9 +153,9 @@ More information about the use of inline caches can be found in Most instructions read or write some data in the form of object references (`PyObject *`). The CPython bytecode interpreter is a stack machine, meaning that its instructions operate by pushing data onto and popping it off the stack. -The stack is forms part of the frame for the code object. Its maximum depth is calculated +The stack forms part of the frame for the code object. Its maximum depth is calculated by the compiler and stored in the `co_stacksize` field of the code object, so that the -stack can be pre-allocated is a contiguous array of `PyObject*` pointers, when the frame +stack can be pre-allocated as a contiguous array of `PyObject*` pointers, when the frame is created. The stack effects of each instruction are also exposed through the @@ -462,7 +462,7 @@ set of values that allows them to: 2. Perform the operation quickly. This requires that the set of values is chosen such that membership can be -tested quickly and that membership is sufficient to allow the operation to +tested quickly and that membership is sufficient to allow the operation to be performed quickly. For example, `LOAD_GLOBAL_MODULE` is specialized for `globals()` From 9d3a8f494985e8bbef698c467099370e233fcbd4 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Sun, 22 Dec 2024 13:01:45 -0600 Subject: [PATCH 270/427] gh-100384: Error on `unguarded-availability` in macOS builds (#128155) Generate a build error on ``unguarded-availability`` in portable macOS builds (i.e. using MACOSX_DEPLOYMENT_TARGET), preventing invalid use of symbols that are not available in older versions of the OS. --- .github/workflows/reusable-macos.yml | 1 + ...-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst | 2 + configure | 41 +++++++++++++++++++ configure.ac | 7 ++++ 4 files changed, 51 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 6fa389b2d66e5e..cdbe05e09fb8e7 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -45,6 +45,7 @@ jobs: brew link --overwrite tcl-tk@8 - name: Configure CPython run: | + MACOSX_DEPLOYMENT_TARGET=10.15 \ GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ ./configure \ diff --git a/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst b/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst new file mode 100644 index 00000000000000..75c19fe3d8cef9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-21-09-56-37.gh-issue-100384.Ib-XrN.rst @@ -0,0 +1,2 @@ +Error on ``unguarded-availability`` in macOS builds, preventing invalid +use of symbols that are not available in older versions of the OS. diff --git a/configure b/configure index e59c7046305d46..a697bc1f87b012 100755 --- a/configure +++ b/configure @@ -10406,6 +10406,47 @@ printf %s "checking which compiler should be used... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 printf "%s\n" "$CC" >&6; } + # Error on unguarded use of new symbols, which will fail at runtime for + # users on older versions of macOS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wunguarded-availability" >&5 +printf %s "checking whether C compiler accepts -Wunguarded-availability... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wunguarded_availability+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wunguarded-availability" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wunguarded_availability=yes +else $as_nop + ax_cv_check_cflags__Werror__Wunguarded_availability=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wunguarded_availability" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wunguarded_availability" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wunguarded_availability" = xyes +then : + as_fn_append CFLAGS_NODIST " -Werror=unguarded-availability" +else $as_nop + : +fi + + LIPO_INTEL64_FLAGS="" if test "${enable_universalsdk}" then diff --git a/configure.ac b/configure.ac index 074e2ce3dd3024..ebc15503f069cc 100644 --- a/configure.ac +++ b/configure.ac @@ -2603,6 +2603,13 @@ AS_VAR_IF([ac_cv_gcc_compat], [yes], [ esac AC_MSG_RESULT([$CC]) + # Error on unguarded use of new symbols, which will fail at runtime for + # users on older versions of macOS + AX_CHECK_COMPILE_FLAG([-Wunguarded-availability], + [AS_VAR_APPEND([CFLAGS_NODIST], [" -Werror=unguarded-availability"])], + [], + [-Werror]) + LIPO_INTEL64_FLAGS="" if test "${enable_universalsdk}" then From 831b6de6d725c697f2f61fd35c4448cd8a9354ff Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Mon, 23 Dec 2024 14:17:19 +1000 Subject: [PATCH 271/427] gh-126180: Remove getopt and optparse deprecation notices (GH-126227) * Remove getopt and optparse deprecation notices * Add new docs sections for command line app helper libraries * Add guidance on choosing a CLI parsing library to the optparse docs * Link to the new guidance from the argparse and getopt docs * Reword intro in docs section for superseded stdlib modules * Reframe the optparse->argparse guide as a migration guide rather than as an upgrade guide --------- Co-authored-by: Serhiy Storchaka --- Doc/howto/argparse-optparse.rst | 36 +++-- Doc/howto/argparse.rst | 15 +- Doc/library/allos.rst | 5 - Doc/library/argparse.rst | 12 ++ Doc/library/cmdlinelibs.rst | 21 +++ Doc/library/filesys.rst | 1 - Doc/library/getopt.rst | 54 +++++-- Doc/library/index.rst | 1 + Doc/library/optparse.rst | 139 ++++++++++++++++-- Doc/library/superseded.rst | 17 ++- Doc/whatsnew/3.13.rst | 24 ++- ...-10-31-14-31-36.gh-issue-126225.vTxGXm.rst | 6 + 12 files changed, 266 insertions(+), 65 deletions(-) create mode 100644 Doc/library/cmdlinelibs.rst create mode 100644 Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst diff --git a/Doc/howto/argparse-optparse.rst b/Doc/howto/argparse-optparse.rst index cef2d893b28a62..b684619885b4c7 100644 --- a/Doc/howto/argparse-optparse.rst +++ b/Doc/howto/argparse-optparse.rst @@ -1,20 +1,14 @@ .. currentmodule:: argparse .. _upgrading-optparse-code: +.. _migrating-optparse-code: -========================== -Upgrading optparse code -========================== +============================================ +Migrating ``optparse`` code to ``argparse`` +============================================ -Originally, the :mod:`argparse` module had attempted to maintain compatibility -with :mod:`optparse`. However, :mod:`optparse` was difficult to extend -transparently, particularly with the changes required to support -``nargs=`` specifiers and better usage messages. When most everything in -:mod:`optparse` had either been copy-pasted over or monkey-patched, it no -longer seemed practical to try to maintain the backwards compatibility. - -The :mod:`argparse` module improves on the :mod:`optparse` -module in a number of ways including: +The :mod:`argparse` module offers several higher level features not natively +provided by the :mod:`optparse` module, including: * Handling positional arguments. * Supporting subcommands. @@ -23,7 +17,23 @@ module in a number of ways including: * Producing more informative usage messages. * Providing a much simpler interface for custom ``type`` and ``action``. -A partial upgrade path from :mod:`optparse` to :mod:`argparse`: +Originally, the :mod:`argparse` module attempted to maintain compatibility +with :mod:`optparse`. However, the fundamental design differences between +supporting declarative command line option processing (while leaving positional +argument processing to application code), and supporting both named options +and positional arguments in the declarative interface mean that the +API has diverged from that of ``optparse`` over time. + +As described in :ref:`choosing-an-argument-parser`, applications that are +currently using :mod:`optparse` and are happy with the way it works can +just continue to use ``optparse``. + +Application developers that are considering migrating should also review +the list of intrinsic behavioural differences described in that section +before deciding whether or not migration is desirable. + +For applications that do choose to migrate from :mod:`optparse` to :mod:`argparse`, +the following suggestions should be helpful: * Replace all :meth:`optparse.OptionParser.add_option` calls with :meth:`ArgumentParser.add_argument` calls. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 1efbee64d60bb3..902c50de00803c 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -13,11 +13,16 @@ recommended command-line parsing module in the Python standard library. .. note:: - There are two other modules that fulfill the same task, namely - :mod:`getopt` (an equivalent for ``getopt()`` from the C - language) and the deprecated :mod:`optparse`. - Note also that :mod:`argparse` is based on :mod:`optparse`, - and therefore very similar in terms of usage. + The standard library includes two other libraries directly related + to command-line parameter processing: the lower level :mod:`optparse` + module (which may require more code to configure for a given application, + but also allows an application to request behaviors that ``argparse`` + doesn't support), and the very low level :mod:`getopt` (which specifically + serves as an equivalent to the :c:func:`!getopt` family of functions + available to C programmers). + While neither of those modules is covered directly in this guide, many of + the core concepts in ``argparse`` first originated in ``optparse``, so + some aspects of this tutorial will also be relevant to ``optparse`` users. Concepts diff --git a/Doc/library/allos.rst b/Doc/library/allos.rst index 0223c1054ea5d8..1aed340b2527ac 100644 --- a/Doc/library/allos.rst +++ b/Doc/library/allos.rst @@ -15,14 +15,9 @@ but they are available on most other systems as well. Here's an overview: os.rst io.rst time.rst - argparse.rst logging.rst logging.config.rst logging.handlers.rst - getpass.rst - curses.rst - curses.ascii.rst - curses.panel.rst platform.rst errno.rst ctypes.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index da4071dee34b8c..8d0116d8c060b8 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -11,6 +11,18 @@ **Source code:** :source:`Lib/argparse.py` +.. note:: + + While :mod:`argparse` is the default recommended standard library module + for implementing basic command line applications, authors with more + exacting requirements for exactly how their command line applications + behave may find it doesn't provide the necessary level of control. + Refer to :ref:`choosing-an-argument-parser` for alternatives to + consider when ``argparse`` doesn't support behaviors that the application + requires (such as entirely disabling support for interspersed options and + positional arguments, or accepting option parameter values that start + with ``-`` even when they correspond to another defined option). + -------------- .. sidebar:: Tutorial diff --git a/Doc/library/cmdlinelibs.rst b/Doc/library/cmdlinelibs.rst new file mode 100644 index 00000000000000..085d31af7bca1f --- /dev/null +++ b/Doc/library/cmdlinelibs.rst @@ -0,0 +1,21 @@ +.. _cmdlinelibs: + +******************************** +Command Line Interface Libraries +******************************** + +The modules described in this chapter assist with implementing +command line and terminal interfaces for applications. + +Here's an overview: + +.. toctree:: + :maxdepth: 1 + + argparse.rst + optparse.rst + getpass.rst + fileinput.rst + curses.rst + curses.ascii.rst + curses.panel.rst diff --git a/Doc/library/filesys.rst b/Doc/library/filesys.rst index 0ccf2b7bf59a0f..f1ea4761af7cb1 100644 --- a/Doc/library/filesys.rst +++ b/Doc/library/filesys.rst @@ -14,7 +14,6 @@ in this chapter is: pathlib.rst os.path.rst - fileinput.rst stat.rst filecmp.rst tempfile.rst diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index 891885d3afbf7a..5c63009e22d58c 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -7,18 +7,13 @@ **Source code:** :source:`Lib/getopt.py` -.. deprecated:: 3.13 - The :mod:`getopt` module is :term:`soft deprecated` and will not be - developed further; development will continue with the :mod:`argparse` - module. - .. note:: - The :mod:`getopt` module is a parser for command line options whose API is - designed to be familiar to users of the C :c:func:`!getopt` function. Users who - are unfamiliar with the C :c:func:`!getopt` function or who would like to write - less code and get better help and error messages should consider using the - :mod:`argparse` module instead. + This module is considered feature complete. A more declarative and + extensible alternative to this API is provided in the :mod:`optparse` + module. Further functional enhancements for command line parameter + processing are provided either as third party modules on PyPI, + or else as features in the :mod:`argparse` module. -------------- @@ -28,6 +23,13 @@ the special meanings of arguments of the form '``-``' and '``--``'). Long options similar to those supported by GNU software may be used as well via an optional third argument. +Users who are unfamiliar with the Unix :c:func:`!getopt` function should consider +using the :mod:`argparse` module instead. Users who are familiar with the Unix +:c:func:`!getopt` function, but would like to get equivalent behavior while +writing less code and getting better help and error messages should consider +using the :mod:`optparse` module. See :ref:`choosing-an-argument-parser` for +additional details. + This module provides two functions and an exception: @@ -194,13 +196,27 @@ In a script, typical usage is something like this: output = a else: assert False, "unhandled option" - # ... + process(args, output=output, verbose=verbose) if __name__ == "__main__": main() Note that an equivalent command line interface could be produced with less code -and more informative help and error messages by using the :mod:`argparse` module: +and more informative help and error messages by using the :mod:`optparse` module: + +.. testcode:: + + import optparse + + if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-o', '--output') + parser.add_option('-v', dest='verbose', action='store_true') + opts, args = parser.parse_args() + process(args, output=opts.output, verbose=opts.verbose) + +A roughly equivalent command line interface for this case can also be +produced by using the :mod:`argparse` module: .. testcode:: @@ -210,12 +226,18 @@ and more informative help and error messages by using the :mod:`argparse` module parser = argparse.ArgumentParser() parser.add_argument('-o', '--output') parser.add_argument('-v', dest='verbose', action='store_true') + parser.add_argument('rest', nargs='*') args = parser.parse_args() - # ... do something with args.output ... - # ... do something with args.verbose .. + process(args.rest, output=args.output, verbose=args.verbose) + +See :ref:`choosing-an-argument-parser` for details on how the ``argparse`` +version of this code differs in behaviour from the ``optparse`` (and +``getopt``) version. .. seealso:: - Module :mod:`argparse` - Alternative command line option and argument parsing library. + Module :mod:`optparse` + Declarative command line option parsing. + Module :mod:`argparse` + More opinionated command line option and argument parsing library. diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 951fbcf13fbb13..44b218948d07e1 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -55,6 +55,7 @@ the `Python Package Index `_. fileformats.rst crypto.rst allos.rst + cmdlinelibs.rst concurrency.rst ipc.rst netdata.rst diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 74a49a8fb33666..ff327cf9162a8c 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -3,25 +3,135 @@ .. module:: optparse :synopsis: Command-line option parsing library. - :deprecated: .. moduleauthor:: Greg Ward .. sectionauthor:: Greg Ward **Source code:** :source:`Lib/optparse.py` -.. deprecated:: 3.2 - The :mod:`optparse` module is :term:`soft deprecated` and will not be - developed further; development will continue with the :mod:`argparse` - module. - -------------- +.. _choosing-an-argument-parser: + +Choosing an argument parsing library +------------------------------------ + +The standard library includes three argument parsing libraries: + +* :mod:`getopt`: a module that closely mirrors the procedural C ``getopt`` API. + Included in the standard library since before the initial Python 1.0 release. +* :mod:`optparse`: a declarative replacement for ``getopt`` that + provides equivalent functionality without requiring each application + to implement its own procedural option parsing logic. Included + in the standard library since the Python 2.3 release. +* :mod:`argparse`: a more opinionated alternative to ``optparse`` that + provides more functionality by default, at the expense of reduced application + flexibility in controlling exactly how arguments are processed. Included in + the standard library since the Python 2.7 and Python 3.2 releases. + +In the absence of more specific argument parsing design constraints, :mod:`argparse` +is the recommended choice for implementing command line applications, as it offers +the highest level of baseline functionality with the least application level code. + +:mod:`getopt` is retained almost entirely for backwards compatibility reasons. +However, it also serves a niche use case as a tool for prototyping and testing +command line argument handling in ``getopt``-based C applications. + +:mod:`optparse` should be considered as an alternative to :mod:`argparse` in the +following cases: + +* an application is already using :mod:`optparse` and doesn't want to risk the + subtle behavioural changes that may arise when migrating to :mod:`argparse` +* the application requires additional control over the way options and + positional parameters are interleaved on the command line (including + the ability to disable the interleaving feature completely) +* the application requires additional control over the incremental parsing + of command line elements (while ``argparse`` does support this, the + exact way it works in practice is undesirable for some use cases) +* the application requires additional control over the handling of options + which accept parameter values that may start with ``-`` (such as delegated + options to be passed to invoked subprocesses) +* the application requires some other command line parameter processing + behavior which ``argparse`` does not support, but which can be implemented + in terms of the lower level interface offered by ``optparse`` + +These considerations also mean that :mod:`optparse` is likely to provide a +better foundation for library authors writing third party command line +argument processing libraries. + +As a concrete example, consider the following two command line argument +parsing configurations, the first using ``optparse``, and the second +using ``argparse``: + +.. testcode:: + + import optparse + + if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-o', '--output') + parser.add_option('-v', dest='verbose', action='store_true') + opts, args = parser.parse_args() + process(args, output=opts.output, verbose=opts.verbose) + +.. testcode:: + + import argparse + + if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-o', '--output') + parser.add_argument('-v', dest='verbose', action='store_true') + parser.add_argument('rest', nargs='*') + args = parser.parse_args() + process(args.rest, output=args.output, verbose=args.verbose) + +The most obvious difference is that in the ``optparse`` version, the non-option +arguments are processed separately by the application after the option processing +is complete. In the ``argparse`` version, positional arguments are declared and +processed in the same way as the named options. + +However, the ``argparse`` version will also handle some parameter combination +differently from the way the ``optparse`` version would handle them. +For example (amongst other differences): + +* supplying ``-o -v`` gives ``output="-v"`` and ``verbose=False`` + when using ``optparse``, but a usage error with ``argparse`` + (complaining that no value has been supplied for ``-o/--output``, + since ``-v`` is interpreted as meaning the verbosity flag) +* similarly, supplying ``-o --`` gives ``output="--"`` and ``args=()`` + when using ``optparse``, but a usage error with ``argparse`` + (also complaining that no value has been supplied for ``-o/--output``, + since ``--`` is interpreted as terminating the option processing + and treating all remaining values as positional arguments) +* supplying ``-o=foo`` gives ``output="=foo"`` when using ``optparse``, + but gives ``output="foo"`` with ``argparse`` (since ``=`` is special + cased as an alternative separator for option parameter values) + +Whether these differing behaviors in the ``argparse`` version are +considered desirable or a problem will depend on the specific command line +application use case. + +.. seealso:: + + :pypi:`click` is a third party argument processing library (originally + based on ``optparse``), which allows command line applications to be + developed as a set of decorated command implementation functions. + + Other third party libraries, such as :pypi:`typer` or :pypi:`msgspec-click`, + allow command line interfaces to be specified in ways that more effectively + integrate with static checking of Python type annotations. + + +Introduction +------------ + :mod:`optparse` is a more convenient, flexible, and powerful library for parsing -command-line options than the old :mod:`getopt` module. :mod:`optparse` uses a -more declarative style of command-line parsing: you create an instance of -:class:`OptionParser`, populate it with options, and parse the command -line. :mod:`optparse` allows users to specify options in the conventional +command-line options than the minimalist :mod:`getopt` module. +:mod:`optparse` uses a more declarative style of command-line parsing: +you create an instance of :class:`OptionParser`, +populate it with options, and parse the command line. +:mod:`optparse` allows users to specify options in the conventional GNU/POSIX syntax, and additionally generates usage and help messages for you. Here's an example of using :mod:`optparse` in a simple script:: @@ -82,10 +192,11 @@ Background ---------- :mod:`optparse` was explicitly designed to encourage the creation of programs -with straightforward, conventional command-line interfaces. To that end, it -supports only the most common command-line syntax and semantics conventionally -used under Unix. If you are unfamiliar with these conventions, read this -section to acquaint yourself with them. +with straightforward command-line interfaces that follow the conventions +established by the :c:func:`!getopt` family of functions available to C developers. +To that end, it supports only the most common command-line syntax and semantics +conventionally used under Unix. If you are unfamiliar with these conventions, +reading this section will allow you to acquaint yourself with them. .. _optparse-terminology: diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index 17bfa66f043302..d120c6acf621e3 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -4,12 +4,23 @@ Superseded Modules ****************** -The modules described in this chapter are deprecated or :term:`soft deprecated` and only kept for -backwards compatibility. They have been superseded by other modules. +The modules described in this chapter have been superseded by other modules +for most use cases, and are retained primarily to preserve backwards compatibility. +Modules may appear in this chapter because they only cover a limited subset of +a problem space, and a more generally applicable solution is available elsewhere +in the standard library (for example, :mod:`getopt` covers the very specific +task of "mimic the C :c:func:`!getopt` API in Python", rather than the broader +command line option parsing and argument parsing capabilities offered by +:mod:`optparse` and :mod:`argparse`). + +Alternatively, modules may appear in this chapter because they are deprecated +outright, and awaiting removal in a future release, or they are +:term:`soft deprecated` and their use is actively discouraged in new projects. +With the removal of various obsolete modules through :pep:`594`, there are +currently no modules in this latter category. .. toctree:: :maxdepth: 1 getopt.rst - optparse.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c8e0f94f4246fb..6a0e483bd895d6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1648,6 +1648,22 @@ opcode (Contributed by Irit Katriel in :gh:`105481`.) +optparse +-------- + +* This module is no longer considered :term:`soft deprecated`. + While :mod:`argparse` remains preferred for new projects that + aren't using a third party command line argument processing + library, there are aspects of the way ``argparse`` works that + mean the lower level ``optparse`` module may provide a better + foundation for *writing* argument processing libraries, and + for implementing command line applications which adhere more + strictly than ``argparse`` does to various Unix command line + processing conventions that originate in the behaviour of the + C :c:func:`!getopt` function . + (Contributed by Alyssa Coghlan and Serhiy Storchaka in :gh:`126180`.) + + pathlib ------- @@ -1787,14 +1803,6 @@ New Deprecations Check membership in :data:`~dis.hasarg` instead. (Contributed by Irit Katriel in :gh:`109319`.) -* :mod:`getopt` and :mod:`optparse`: - - * Both modules are now :term:`soft deprecated`, - with :mod:`argparse` preferred for new projects. - This is a new soft-deprecation for the :mod:`!getopt` module, - whereas the :mod:`!optparse` module was already *de facto* soft deprecated. - (Contributed by Victor Stinner in :gh:`106535`.) - * :mod:`gettext`: * Deprecate non-integer numbers as arguments to functions and methods diff --git a/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst b/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst new file mode 100644 index 00000000000000..13a1f213c7a58e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-31-14-31-36.gh-issue-126225.vTxGXm.rst @@ -0,0 +1,6 @@ +:mod:`getopt` and :mod:`optparse` are no longer marked as deprecated. +There are legitimate reasons to use one of these modules in preference to +:mod:`argparse`, and none of these modules are at risk of being removed +from the standard library. Of the three, ``argparse`` remains the +recommended default choice, *unless* one of the concerns noted at the top of +the ``optparse`` module documentation applies. From 180d417e9f9456defd4c5b53cae678c318287921 Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Mon, 23 Dec 2024 04:31:33 -0800 Subject: [PATCH 272/427] gh-114203: Optimise simple recursive critical sections (#128126) Add a fast path to (single-mutex) critical section locking _iff_ the mutex is already held by the currently active, top-most critical section of this thread. This can matter a lot for indirectly recursive critical sections without intervening critical sections. --- Include/internal/pycore_critical_section.h | 14 +++++++++++ ...-12-20-23-07-33.gh-issue-114203.84NgoW.rst | 1 + Python/critical_section.c | 24 +++++++++++++------ 3 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 9ba2fce56d3c9c..e66d6d805c1b3b 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -145,6 +145,12 @@ _PyCriticalSection_Pop(PyCriticalSection *c) static inline void _PyCriticalSection_End(PyCriticalSection *c) { + // If the mutex is NULL, we used the fast path in + // _PyCriticalSection_BeginSlow for locks already held in the top-most + // critical section, and we shouldn't unlock or pop this critical section. + if (c->_cs_mutex == NULL) { + return; + } PyMutex_Unlock(c->_cs_mutex); _PyCriticalSection_Pop(c); } @@ -199,6 +205,14 @@ _PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) static inline void _PyCriticalSection2_End(PyCriticalSection2 *c) { + // if mutex1 is NULL, we used the fast path in + // _PyCriticalSection_BeginSlow for mutexes that are already held, + // which should only happen when mutex1 and mutex2 were the same mutex, + // and mutex2 should also be NULL. + if (c->_cs_base._cs_mutex == NULL) { + assert(c->_cs_mutex2 == NULL); + return; + } if (c->_cs_mutex2) { PyMutex_Unlock(c->_cs_mutex2); } diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst new file mode 100644 index 00000000000000..6a9856e90c32bc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-20-23-07-33.gh-issue-114203.84NgoW.rst @@ -0,0 +1 @@ +Optimize ``Py_BEGIN_CRITICAL_SECTION`` for simple recursive calls. diff --git a/Python/critical_section.c b/Python/critical_section.c index 62ed25523fd6dc..73857b85496316 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -8,11 +8,28 @@ static_assert(_Alignof(PyCriticalSection) >= 4, "critical section must be aligned to at least 4 bytes"); #endif +#ifdef Py_GIL_DISABLED +static PyCriticalSection * +untag_critical_section(uintptr_t tag) +{ + return (PyCriticalSection *)(tag & ~_Py_CRITICAL_SECTION_MASK); +} +#endif + void _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) { #ifdef Py_GIL_DISABLED PyThreadState *tstate = _PyThreadState_GET(); + // As an optimisation for locking the same object recursively, skip + // locking if the mutex is currently locked by the top-most critical + // section. + if (tstate->critical_section && + untag_critical_section(tstate->critical_section)->_cs_mutex == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } c->_cs_mutex = NULL; c->_cs_prev = (uintptr_t)tstate->critical_section; tstate->critical_section = (uintptr_t)c; @@ -42,13 +59,6 @@ _PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, #endif } -#ifdef Py_GIL_DISABLED -static PyCriticalSection * -untag_critical_section(uintptr_t tag) -{ - return (PyCriticalSection *)(tag & ~_Py_CRITICAL_SECTION_MASK); -} -#endif // Release all locks held by critical sections. This is called by // _PyThreadState_Detach. From c5b0c90b62f1a10b0742db4bcd17da080d4e9111 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 24 Dec 2024 02:08:34 +0900 Subject: [PATCH 273/427] gh-115999: Update test_opcache to test with nested method (gh-128166) gh-115999: Update test_opcace to test with nested method --- Lib/test/test_opcache.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index ba111b5117b41d..79f452f8068c7f 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -606,7 +606,7 @@ def assert_races_do_not_crash( for writer in writers: writer.join() - @requires_specialization + @requires_specialization_ft def test_binary_subscr_getitem(self): def get_items(): class C: @@ -1242,14 +1242,6 @@ def f(o, n): f(test_obj, 1) self.assertEqual(test_obj.b, 0) -# gh-127274: BINARY_SUBSCR_GETITEM will only cache __getitem__ methods that -# are deferred. We only defer functions defined at the top-level. -class CGetItem: - def __init__(self, val): - self.val = val - def __getitem__(self, item): - return self.val - class TestSpecializer(TestBase): @@ -1592,7 +1584,13 @@ def binary_subscr_str_int(): self.assert_no_opcode(binary_subscr_str_int, "BINARY_SUBSCR") def binary_subscr_getitems(): - items = [CGetItem(i) for i in range(100)] + class C: + def __init__(self, val): + self.val = val + def __getitem__(self, item): + return self.val + + items = [C(i) for i in range(100)] for i in range(100): self.assertEqual(items[i][i], i) From d61542b5ff1fe64705e5ce1bcc53048f14098dba Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Mon, 23 Dec 2024 17:22:15 +0000 Subject: [PATCH 274/427] pathlib tests: create test hierarchy without using class under test (#128200) In the pathlib tests, avoid using the path class under test (`self.cls`) in test setup. Instead we use `os` functions in `test_pathlib`, and direct manipulation of `DummyPath` internal data in `test_pathlib_abc`. --- Lib/test/test_pathlib/test_pathlib.py | 38 ++++++++++++++++++- Lib/test/test_pathlib/test_pathlib_abc.py | 46 ++++++++++------------- 2 files changed, 55 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index ef482c311542fa..fac8cbdf65a122 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -578,10 +578,44 @@ def setUp(self): if name in _tests_needing_symlinks and not self.can_symlink: self.skipTest('requires symlinks') super().setUp() - os.chmod(self.parser.join(self.base, 'dirE'), 0) + + def createTestHierarchy(self): + os.mkdir(self.base) + os.mkdir(os.path.join(self.base, 'dirA')) + os.mkdir(os.path.join(self.base, 'dirB')) + os.mkdir(os.path.join(self.base, 'dirC')) + os.mkdir(os.path.join(self.base, 'dirC', 'dirD')) + os.mkdir(os.path.join(self.base, 'dirE')) + with open(os.path.join(self.base, 'fileA'), 'wb') as f: + f.write(b"this is file A\n") + with open(os.path.join(self.base, 'dirB', 'fileB'), 'wb') as f: + f.write(b"this is file B\n") + with open(os.path.join(self.base, 'dirC', 'fileC'), 'wb') as f: + f.write(b"this is file C\n") + with open(os.path.join(self.base, 'dirC', 'novel.txt'), 'wb') as f: + f.write(b"this is a novel\n") + with open(os.path.join(self.base, 'dirC', 'dirD', 'fileD'), 'wb') as f: + f.write(b"this is file D\n") + os.chmod(os.path.join(self.base, 'dirE'), 0) + if self.can_symlink: + # Relative symlinks. + os.symlink('fileA', os.path.join(self.base, 'linkA')) + os.symlink('non-existing', os.path.join(self.base, 'brokenLink')) + os.symlink('dirB', + os.path.join(self.base, 'linkB'), + target_is_directory=True) + os.symlink(os.path.join('..', 'dirB'), + os.path.join(self.base, 'dirA', 'linkC'), + target_is_directory=True) + # This one goes upwards, creating a loop. + os.symlink(os.path.join('..', 'dirB'), + os.path.join(self.base, 'dirB', 'linkD'), + target_is_directory=True) + # Broken symlink (pointing to itself). + os.symlink('brokenLinkLoop', os.path.join(self.base, 'brokenLinkLoop')) def tearDown(self): - os.chmod(self.parser.join(self.base, 'dirE'), 0o777) + os.chmod(os.path.join(self.base, 'dirE'), 0o777) os_helper.rmtree(self.base) def tempdir(self): diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 9198a0cbc45cee..787fb0f82257e6 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1437,33 +1437,25 @@ class DummyPathTest(DummyPurePathTest): def setUp(self): super().setUp() - parser = self.cls.parser - p = self.cls(self.base) - p.mkdir(parents=True) - p.joinpath('dirA').mkdir() - p.joinpath('dirB').mkdir() - p.joinpath('dirC').mkdir() - p.joinpath('dirC', 'dirD').mkdir() - p.joinpath('dirE').mkdir() - with p.joinpath('fileA').open('wb') as f: - f.write(b"this is file A\n") - with p.joinpath('dirB', 'fileB').open('wb') as f: - f.write(b"this is file B\n") - with p.joinpath('dirC', 'fileC').open('wb') as f: - f.write(b"this is file C\n") - with p.joinpath('dirC', 'novel.txt').open('wb') as f: - f.write(b"this is a novel\n") - with p.joinpath('dirC', 'dirD', 'fileD').open('wb') as f: - f.write(b"this is file D\n") - if self.can_symlink: - p.joinpath('linkA').symlink_to('fileA') - p.joinpath('brokenLink').symlink_to('non-existing') - p.joinpath('linkB').symlink_to('dirB', target_is_directory=True) - p.joinpath('dirA', 'linkC').symlink_to( - parser.join('..', 'dirB'), target_is_directory=True) - p.joinpath('dirB', 'linkD').symlink_to( - parser.join('..', 'dirB'), target_is_directory=True) - p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') + self.createTestHierarchy() + + def createTestHierarchy(self): + cls = self.cls + cls._files = { + f'{self.base}/fileA': b'this is file A\n', + f'{self.base}/dirB/fileB': b'this is file B\n', + f'{self.base}/dirC/fileC': b'this is file C\n', + f'{self.base}/dirC/dirD/fileD': b'this is file D\n', + f'{self.base}/dirC/novel.txt': b'this is a novel\n', + } + cls._directories = { + f'{self.base}': {'fileA', 'dirA', 'dirB', 'dirC', 'dirE'}, + f'{self.base}/dirA': set(), + f'{self.base}/dirB': {'fileB'}, + f'{self.base}/dirC': {'fileC', 'dirD', 'novel.txt'}, + f'{self.base}/dirC/dirD': {'fileD'}, + f'{self.base}/dirE': set(), + } def tearDown(self): cls = self.cls From 30efede33ca1fe32debbae93cc40b0e7e0b133b3 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Mon, 23 Dec 2024 22:17:47 +0100 Subject: [PATCH 275/427] gh-128195: Add `_REPLACE_WITH_TRUE` to the tier2 optimizer (GH-128203) Add `_REPLACE_WITH_TRUE` to the tier2 optimizer --- Python/optimizer_bytecodes.c | 4 ++++ Python/optimizer_cases.c.h | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e60c0d38425bfe..a14d119b7a1dec 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -899,6 +899,10 @@ dummy_func(void) { (void)version; } + op(_REPLACE_WITH_TRUE, (value -- res)) { + res = sym_new_const(ctx, Py_True); + } + // END BYTECODES // } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b46079ec8a1992..0fcf5e18ed5808 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -211,7 +211,7 @@ case _REPLACE_WITH_TRUE: { _Py_UopsSymbol *res; - res = sym_new_not_null(ctx); + res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; break; } From 3f6a618e49b1c8c12a7bc0c26e846735e108dc97 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 24 Dec 2024 09:43:31 +0000 Subject: [PATCH 276/427] gh-127949: fix `DeprecationWarning` in test_inspect.py (#128215) --- Lib/test/test_inspect/test_inspect.py | 28 +++++++++++++-------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 2c950e46b3ed8a..d536d04d2e7d88 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1,5 +1,4 @@ from annotationlib import Format, ForwardRef -import asyncio import builtins import collections import copy @@ -73,11 +72,6 @@ def revise(filename, *args): git = mod.StupidGit() -def tearDownModule(): - if support.has_socket_support: - asyncio._set_event_loop_policy(None) - - def signatures_with_lexicographic_keyword_only_parameters(): """ Yields a whole bunch of functions with only keyword-only parameters, @@ -205,7 +199,7 @@ def test_excluding_predicates(self): self.assertFalse(inspect.ismethodwrapper(type("AnyClass", (), {}))) def test_ispackage(self): - self.istest(inspect.ispackage, 'asyncio') + self.istest(inspect.ispackage, 'unittest') self.istest(inspect.ispackage, 'importlib') self.assertFalse(inspect.ispackage(inspect)) self.assertFalse(inspect.ispackage(mod)) @@ -1166,16 +1160,20 @@ def f(self): # This is necessary when the test is run multiple times. sys.modules.pop("inspect_actual") - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "socket.accept is broken" - ) def test_nested_class_definition_inside_async_function(self): - import asyncio - self.addCleanup(asyncio.set_event_loop_policy, None) - self.assertSourceEqual(asyncio.run(mod2.func225()), 226, 227) + def run(coro): + try: + coro.send(None) + except StopIteration as e: + return e.value + else: + raise RuntimeError("coroutine did not complete synchronously!") + finally: + coro.close() + + self.assertSourceEqual(run(mod2.func225()), 226, 227) self.assertSourceEqual(mod2.cls226, 231, 235) - self.assertSourceEqual(asyncio.run(mod2.cls226().func232()), 233, 234) + self.assertSourceEqual(run(mod2.cls226().func232()), 233, 234) def test_class_definition_same_name_diff_methods(self): self.assertSourceEqual(mod2.cls296, 296, 298) From a391d80f4bf5a3cf5aa95340ca848b9a0294778d Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 24 Dec 2024 17:30:26 +0530 Subject: [PATCH 277/427] gh-127949: deprecate asyncio policy classes (#128216) --- Doc/library/asyncio-policy.rst | 16 +++++++++ Lib/asyncio/__init__.py | 16 +++++++++ Lib/asyncio/events.py | 14 ++++---- Lib/asyncio/unix_events.py | 6 ++-- Lib/asyncio/windows_events.py | 10 +++--- Lib/test/test_asyncio/test_events.py | 34 +++++++++++++------- Lib/test/test_asyncio/test_runners.py | 2 +- Lib/test/test_asyncio/test_windows_events.py | 18 +++++++---- 8 files changed, 82 insertions(+), 34 deletions(-) diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 2d05c3a9f7f157..ea7fe957da7c40 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -87,6 +87,10 @@ The abstract event loop policy base class is defined as follows: This method should never return ``None``. + .. deprecated:: next + The :class:`AbstractEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. _asyncio-policy-builtin: @@ -109,6 +113,10 @@ asyncio ships with the following built-in policies: The :meth:`get_event_loop` method of the default asyncio policy now raises a :exc:`RuntimeError` if there is no set event loop. + .. deprecated:: next + The :class:`DefaultEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. class:: WindowsSelectorEventLoopPolicy @@ -117,6 +125,10 @@ asyncio ships with the following built-in policies: .. availability:: Windows. + .. deprecated:: next + The :class:`WindowsSelectorEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. class:: WindowsProactorEventLoopPolicy @@ -125,6 +137,10 @@ asyncio ships with the following built-in policies: .. availability:: Windows. + .. deprecated:: next + The :class:`WindowsProactorEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. _asyncio-custom-policies: diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 03165a425eb7d2..edb615b1b6b1c6 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -45,3 +45,19 @@ else: from .unix_events import * # pragma: no cover __all__ += unix_events.__all__ + +def __getattr__(name: str): + import warnings + + deprecated = { + "AbstractEventLoopPolicy", + "DefaultEventLoopPolicy", + "WindowsSelectorEventLoopPolicy", + "WindowsProactorEventLoopPolicy", + } + if name in deprecated: + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + # deprecated things have underscores in front of them + return globals()["_" + name] + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 1449245edc7c7e..3ade7747149b32 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,7 +5,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - 'AbstractEventLoopPolicy', + '_AbstractEventLoopPolicy', 'AbstractEventLoop', 'AbstractServer', 'Handle', 'TimerHandle', '_get_event_loop_policy', @@ -632,7 +632,7 @@ def set_debug(self, enabled): raise NotImplementedError -class AbstractEventLoopPolicy: +class _AbstractEventLoopPolicy: """Abstract policy for accessing the event loop.""" def get_event_loop(self): @@ -655,7 +655,7 @@ def new_event_loop(self): the current context, set_event_loop must be called explicitly.""" raise NotImplementedError -class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): +class _BaseDefaultEventLoopPolicy(_AbstractEventLoopPolicy): """Default policy implementation for accessing the event loop. In this policy, each thread has its own event loop. However, we @@ -758,8 +758,8 @@ def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import DefaultEventLoopPolicy - _event_loop_policy = DefaultEventLoopPolicy() + from . import _DefaultEventLoopPolicy + _event_loop_policy = _DefaultEventLoopPolicy() def _get_event_loop_policy(): @@ -777,7 +777,7 @@ def _set_event_loop_policy(policy): If policy is None, the default policy is restored.""" global _event_loop_policy - if policy is not None and not isinstance(policy, AbstractEventLoopPolicy): + if policy is not None and not isinstance(policy, _AbstractEventLoopPolicy): raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") _event_loop_policy = policy @@ -838,7 +838,7 @@ def new_event_loop(): def on_fork(): # Reset the loop and wakeupfd in the forked child process. if _event_loop_policy is not None: - _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local() + _event_loop_policy._local = _BaseDefaultEventLoopPolicy._Local() _set_running_loop(None) signal.set_wakeup_fd(-1) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 0227eb506c6016..f69c6a64c39ae6 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,7 +28,7 @@ __all__ = ( 'SelectorEventLoop', - 'DefaultEventLoopPolicy', + '_DefaultEventLoopPolicy', 'EventLoop', ) @@ -963,11 +963,11 @@ def can_use_pidfd(): return True -class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy): """UNIX event loop policy""" _loop_factory = _UnixSelectorEventLoop SelectorEventLoop = _UnixSelectorEventLoop -DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy +_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy EventLoop = SelectorEventLoop diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index bf99bc271c7acd..5f75b17d8ca649 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -29,8 +29,8 @@ __all__ = ( 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - 'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy', - 'WindowsProactorEventLoopPolicy', 'EventLoop', + '_DefaultEventLoopPolicy', '_WindowsSelectorEventLoopPolicy', + '_WindowsProactorEventLoopPolicy', 'EventLoop', ) @@ -891,13 +891,13 @@ def callback(f): SelectorEventLoop = _WindowsSelectorEventLoop -class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsSelectorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = SelectorEventLoop -class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsProactorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = ProactorEventLoop -DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy +_DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy EventLoop = ProactorEventLoop diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index d43f66c13d2f96..c626670f72a084 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2695,14 +2695,26 @@ async def inner(): class PolicyTests(unittest.TestCase): + def test_abstract_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.AbstractEventLoopPolicy' is deprecated"): + policy = asyncio.AbstractEventLoopPolicy() + self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) + + def test_default_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.DefaultEventLoopPolicy' is deprecated"): + policy = asyncio.DefaultEventLoopPolicy() + self.assertIsInstance(policy, asyncio.DefaultEventLoopPolicy) + def test_event_loop_policy(self): - policy = asyncio.AbstractEventLoopPolicy() + policy = asyncio._AbstractEventLoopPolicy() self.assertRaises(NotImplementedError, policy.get_event_loop) self.assertRaises(NotImplementedError, policy.set_event_loop, object()) self.assertRaises(NotImplementedError, policy.new_event_loop) def test_get_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) with self.assertRaises(RuntimeError): @@ -2710,7 +2722,7 @@ def test_get_event_loop(self): self.assertIsNone(policy._local._loop) def test_get_event_loop_does_not_call_set_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() with mock.patch.object( policy, "set_event_loop", @@ -2722,7 +2734,7 @@ def test_get_event_loop_does_not_call_set_event_loop(self): m_set_event_loop.assert_not_called() def test_get_event_loop_after_set_none(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() policy.set_event_loop(None) self.assertRaises(RuntimeError, policy.get_event_loop) @@ -2730,7 +2742,7 @@ def test_get_event_loop_after_set_none(self): def test_get_event_loop_thread(self, m_current_thread): def f(): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() self.assertRaises(RuntimeError, policy.get_event_loop) th = threading.Thread(target=f) @@ -2738,14 +2750,14 @@ def f(): th.join() def test_new_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() loop = policy.new_event_loop() self.assertIsInstance(loop, asyncio.AbstractEventLoop) loop.close() def test_set_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() old_loop = policy.new_event_loop() policy.set_event_loop(old_loop) @@ -2762,7 +2774,7 @@ def test_get_event_loop_policy(self): with self.assertWarnsRegex( DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): policy = asyncio.get_event_loop_policy() - self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) + self.assertIsInstance(policy, asyncio._AbstractEventLoopPolicy) self.assertIs(policy, asyncio.get_event_loop_policy()) def test_set_event_loop_policy(self): @@ -2775,7 +2787,7 @@ def test_set_event_loop_policy(self): DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): old_policy = asyncio.get_event_loop_policy() - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() with self.assertWarnsRegex( DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): asyncio.set_event_loop_policy(policy) @@ -2862,7 +2874,7 @@ def test_get_event_loop_returns_running_loop(self): class TestError(Exception): pass - class Policy(asyncio.DefaultEventLoopPolicy): + class Policy(asyncio._DefaultEventLoopPolicy): def get_event_loop(self): raise TestError @@ -2908,7 +2920,7 @@ async def func(): def test_get_event_loop_returns_running_loop2(self): old_policy = asyncio._get_event_loop_policy() try: - asyncio._set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) + asyncio._set_event_loop_policy(asyncio._DefaultEventLoopPolicy()) loop = asyncio.new_event_loop() self.addCleanup(loop.close) diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index e1f82f7f7bec0c..21f277bc2d8d5f 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -19,7 +19,7 @@ def interrupt_self(): _thread.interrupt_main() -class TestPolicy(asyncio.AbstractEventLoopPolicy): +class TestPolicy(asyncio._AbstractEventLoopPolicy): def __init__(self, loop_factory): self.loop_factory = loop_factory diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 28b05d24dc25a1..69e9905205eee0 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -328,14 +328,15 @@ class WinPolicyTests(WindowsEventsTestCase): def test_selector_win_policy(self): async def main(): - self.assertIsInstance( - asyncio.get_running_loop(), - asyncio.SelectorEventLoop) + self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) old_policy = asyncio._get_event_loop_policy() try: - asyncio._set_event_loop_policy( - asyncio.WindowsSelectorEventLoopPolicy()) + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsSelectorEventLoopPolicy' is deprecated", + ): + asyncio._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.run(main()) finally: asyncio._set_event_loop_policy(old_policy) @@ -348,8 +349,11 @@ async def main(): old_policy = asyncio._get_event_loop_policy() try: - asyncio._set_event_loop_policy( - asyncio.WindowsProactorEventLoopPolicy()) + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsProactorEventLoopPolicy' is deprecated", + ): + asyncio._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) asyncio.run(main()) finally: asyncio._set_event_loop_policy(old_policy) From bd3e200cce6601684108e23e7621bbe05b53a323 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Tue, 24 Dec 2024 08:01:21 -0500 Subject: [PATCH 278/427] Add Windows version comments to the python manifest. (GH-127439) --- PC/python.manifest | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/PC/python.manifest b/PC/python.manifest index 8e1bc022adfb4f..19c9fc1b80a3ec 100644 --- a/PC/python.manifest +++ b/PC/python.manifest @@ -9,10 +9,15 @@ + + + + + From 3ddd70ceaaf67b111ee4251817e150396d6d10a9 Mon Sep 17 00:00:00 2001 From: Sergey Muraviov Date: Tue, 24 Dec 2024 16:06:41 +0300 Subject: [PATCH 279/427] gh-128217: Validate the normalized_environment variable instead of the similarly named function (GH-128220) --- Modules/_winapi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 4ce689fe30e6df..260cab48091c16 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1048,7 +1048,7 @@ getenvironment(PyObject* environment) } normalized_environment = normalize_environment(environment); - if (normalize_environment == NULL) { + if (normalized_environment == NULL) { return NULL; } From 9fce90682553e2cfe93e98e2ae5948bf9c7c4456 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 24 Dec 2024 19:24:28 +0530 Subject: [PATCH 280/427] gh-127949: deprecate `asyncio.set_event_loop` (#128218) Deprecate `asyncio.set_event_loop` to be removed in Python 3.16. --- Doc/library/asyncio-eventloop.rst | 4 +++ Lib/asyncio/__main__.py | 2 +- Lib/asyncio/events.py | 32 +++++++++++++++-------- Lib/asyncio/runners.py | 4 +-- Lib/test/test_asyncgen.py | 2 +- Lib/test/test_asyncio/functional.py | 4 +-- Lib/test/test_asyncio/test_base_events.py | 8 +++--- Lib/test/test_asyncio/test_events.py | 26 +++++++++++------- Lib/test/test_asyncio/test_futures.py | 8 +++--- Lib/test/test_asyncio/test_streams.py | 10 +++---- Lib/test/test_asyncio/test_tasks.py | 18 ++++++------- Lib/test/test_asyncio/test_unix_events.py | 4 +-- Lib/test/test_asyncio/utils.py | 4 +-- Lib/test/test_coroutines.py | 2 +- Lib/test/test_type_params.py | 2 +- Lib/test/test_unittest/test_async_case.py | 2 +- 16 files changed, 77 insertions(+), 55 deletions(-) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 9f1aec148f8750..29f843123f8560 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -66,6 +66,10 @@ an event loop: Set *loop* as the current event loop for the current OS thread. + .. deprecated:: next + The :func:`set_event_loop` function is deprecated and will be removed + in Python 3.16. + .. function:: new_event_loop() Create and return a new event loop object. diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 95c636f9e02866..662ba649aa08be 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -149,7 +149,7 @@ def interrupt(self) -> None: return_code = 0 loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) repl_locals = {'asyncio': asyncio} for key in {'__name__', '__package__', diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 3ade7747149b32..6e291d28ec81ae 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,16 +5,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - '_AbstractEventLoopPolicy', - 'AbstractEventLoop', 'AbstractServer', - 'Handle', 'TimerHandle', - '_get_event_loop_policy', - 'get_event_loop_policy', - '_set_event_loop_policy', - 'set_event_loop_policy', - 'get_event_loop', 'set_event_loop', 'new_event_loop', - '_set_running_loop', 'get_running_loop', - '_get_running_loop', + "_AbstractEventLoopPolicy", + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "_get_event_loop_policy", + "get_event_loop_policy", + "_set_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "_set_event_loop", + "set_event_loop", + "new_event_loop", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", ) import contextvars @@ -801,9 +807,13 @@ def get_event_loop(): return _get_event_loop_policy().get_event_loop() +def _set_event_loop(loop): + _get_event_loop_policy().set_event_loop(loop) + def set_event_loop(loop): """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - _get_event_loop_policy().set_event_loop(loop) + warnings._deprecated('asyncio.set_event_loop', remove=(3,16)) + _set_event_loop(loop) def new_event_loop(): diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 0e63c34f60f4d9..b9adf291d4817f 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -74,7 +74,7 @@ def close(self): loop.shutdown_default_executor(constants.THREAD_JOIN_TIMEOUT)) finally: if self._set_event_loop: - events.set_event_loop(None) + events._set_event_loop(None) loop.close() self._loop = None self._state = _State.CLOSED @@ -147,7 +147,7 @@ def _lazy_init(self): if not self._set_event_loop: # Call set_event_loop only once to avoid calling # attach_loop multiple times on child watchers - events.set_event_loop(self._loop) + events._set_event_loop(self._loop) self._set_event_loop = True else: self._loop = self._loop_factory() diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 4bce6d5c1b1d2f..5bfd789185c675 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -624,7 +624,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) def tearDown(self): self.loop.close() diff --git a/Lib/test/test_asyncio/functional.py b/Lib/test/test_asyncio/functional.py index d19c7a612ccf86..2934325b6dfbc7 100644 --- a/Lib/test/test_asyncio/functional.py +++ b/Lib/test/test_asyncio/functional.py @@ -24,7 +24,7 @@ def loop_exception_handler(self, loop, context): def setUp(self): self.loop = self.new_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) self.loop.set_exception_handler(self.loop_exception_handler) self.__unhandled_exceptions = [] @@ -39,7 +39,7 @@ def tearDown(self): self.fail('unexpected calls to loop.call_exception_handler()') finally: - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) self.loop = None def tcp_server(self, server_prog, *, diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 08e38b047d519b..1e063c1352ecb9 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -331,10 +331,10 @@ def check_in_thread(loop, event, debug, create_loop, fut): if create_loop: loop2 = base_events.BaseEventLoop() try: - asyncio.set_event_loop(loop2) + asyncio._set_event_loop(loop2) self.check_thread(loop, debug) finally: - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) loop2.close() else: self.check_thread(loop, debug) @@ -690,7 +690,7 @@ def default_exception_handler(self, context): loop = Loop() self.addCleanup(loop.close) - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) def run_loop(): def zero_error(): @@ -1983,7 +1983,7 @@ def stop_loop_cb(loop): async def stop_loop_coro(loop): loop.stop() - asyncio.set_event_loop(self.loop) + asyncio._set_event_loop(self.loop) self.loop.set_debug(True) self.loop.slow_callback_duration = 0.0 diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index c626670f72a084..c8439c9af5e6ba 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -58,7 +58,7 @@ async def doit(): return 'hello' loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) return loop.run_until_complete(doit()) @@ -2695,6 +2695,14 @@ async def inner(): class PolicyTests(unittest.TestCase): + def test_asyncio_set_event_loop_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop' is deprecated"): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self.assertIs(loop, asyncio.get_event_loop()) + loop.close() + def test_abstract_event_loop_policy_deprecation(self): with self.assertWarnsRegex( DeprecationWarning, "'asyncio.AbstractEventLoopPolicy' is deprecated"): @@ -2824,14 +2832,14 @@ def setUp(self): super().setUp() self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) + asyncio._set_event_loop(self.loop) def tearDown(self): try: super().tearDown() finally: self.loop.close() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) events._get_running_loop = self._get_running_loop_saved events._set_running_loop = self._set_running_loop_saved @@ -2885,7 +2893,7 @@ def get_event_loop(self): with self.assertRaises(TestError): asyncio.get_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaises(TestError): asyncio.get_event_loop() @@ -2900,10 +2908,10 @@ async def func(): loop.run_until_complete(func()) - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) with self.assertRaises(TestError): asyncio.get_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaises(TestError): asyncio.get_event_loop() @@ -2927,7 +2935,7 @@ def test_get_event_loop_returns_running_loop2(self): with self.assertRaisesRegex(RuntimeError, 'no current'): asyncio.get_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaisesRegex(RuntimeError, 'no current'): asyncio.get_event_loop() @@ -2938,10 +2946,10 @@ async def func(): loop.run_until_complete(func()) - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) self.assertIs(asyncio.get_event_loop(), loop) - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaisesRegex(RuntimeError, 'no current'): asyncio.get_event_loop() diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 7db70a4c81d483..84b44011b9a844 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -178,8 +178,8 @@ async def test(): def test_constructor_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) f = self._new_future() self.assertIs(f._loop, self.loop) self.assertIs(f.get_loop(), self.loop) @@ -566,8 +566,8 @@ async def test(): def test_wrap_future_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index c3ba90b309e49f..047ada8c5d23df 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -71,7 +71,7 @@ def _basetest_open_connection_no_loop_ssl(self, open_connection_fut): try: reader, writer = self.loop.run_until_complete(open_connection_fut) finally: - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) writer.write(b'GET / HTTP/1.0\r\n\r\n') f = reader.read() data = self.loop.run_until_complete(f) @@ -839,8 +839,8 @@ def test_streamreader_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set # Deprecated in 3.10, undeprecated in 3.12 - self.addCleanup(asyncio.set_event_loop, None) - asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) + asyncio._set_event_loop(self.loop) reader = asyncio.StreamReader() self.assertIs(reader._loop, self.loop) @@ -863,8 +863,8 @@ def test_streamreaderprotocol_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set # Deprecated in 3.10, undeprecated in 3.12 - self.addCleanup(asyncio.set_event_loop, None) - asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) + asyncio._set_event_loop(self.loop) reader = mock.Mock() protocol = asyncio.StreamReaderProtocol(reader) self.assertIs(protocol._loop, self.loop) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 7d6d0564a9a9db..b5363226ad79f4 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -212,8 +212,8 @@ async def test(): self.assertEqual(t.result(), 'ok') # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) @@ -2202,8 +2202,8 @@ def test_shield_coroutine_use_global_loop(self): async def coro(): return 42 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) outer = asyncio.shield(coro()) self.assertEqual(outer._loop, self.loop) res = self.loop.run_until_complete(outer) @@ -2273,7 +2273,7 @@ async def kill_me(loop): self.assertEqual(self.all_tasks(loop=self.loop), {task}) - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) # execute the task so it waits for future self.loop._run_once() @@ -3278,8 +3278,8 @@ async def gather(): def test_constructor_empty_sequence_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.one_loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.one_loop) + self.addCleanup(asyncio._set_event_loop, None) fut = asyncio.gather() self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) @@ -3386,8 +3386,8 @@ def test_constructor_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 async def coro(): return 'abc' - asyncio.set_event_loop(self.other_loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.other_loop) + self.addCleanup(asyncio._set_event_loop, None) gen1 = coro() gen2 = coro() fut = asyncio.gather(gen1, gen2) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index e9ee9702248015..ebb4cc0f7b64fd 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1116,11 +1116,11 @@ class TestFunctional(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) + asyncio._set_event_loop(self.loop) def tearDown(self): self.loop.close() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) def test_add_reader_invalid_argument(self): def assert_raises(): diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index b8dbe7feaac3f4..35ce13896da08f 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -541,7 +541,7 @@ def set_event_loop(self, loop, *, cleanup=True): if loop is None: raise AssertionError('loop is None') # ensure that the event loop is passed explicitly in asyncio - events.set_event_loop(None) + events._set_event_loop(None) if cleanup: self.addCleanup(self.close_loop, loop) @@ -554,7 +554,7 @@ def setUp(self): self._thread_cleanup = threading_helper.threading_setup() def tearDown(self): - events.set_event_loop(None) + events._set_event_loop(None) # Detect CPython bug #23353: ensure that yield/yield-from is not used # in an except block of a generator diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index a72c43f9b47947..840043d5271224 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2287,7 +2287,7 @@ async def f(): buffer.append('unreachable') loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) try: loop.run_until_complete(f()) except MyException: diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 433b19593bdd04..89f836cf722966 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1060,7 +1060,7 @@ async def coroutine[B](): co = get_coroutine() - self.addCleanup(asyncio.set_event_loop_policy, None) + self.addCleanup(asyncio._set_event_loop_policy, None) a, b = asyncio.run(co()) self.assertIsInstance(a, TypeVar) diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 993e6bf013cfbf..fc996b42149dcb 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -476,7 +476,7 @@ async def cleanup(self, fut): def test_setup_get_event_loop(self): # See https://github.com/python/cpython/issues/95736 # Make sure the default event loop is not used - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) class TestCase1(unittest.IsolatedAsyncioTestCase): def setUp(self): From 7ed6c5c6961d0849f163d4d449fb36bae312b6bc Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Tue, 24 Dec 2024 07:56:42 -0800 Subject: [PATCH 281/427] gh-127847: Fix position in the special-cased zipfile seek (#127856) --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Peter Bierma Co-authored-by: Jason R. Coombs --- Lib/test/test_zipfile/test_core.py | 12 ++++++++++++ Lib/zipfile/__init__.py | 5 ++++- .../2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index c36228c033a414..124e088fd15b80 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -2333,6 +2333,18 @@ def test_read_after_seek(self): fp.seek(1, os.SEEK_CUR) self.assertEqual(fp.read(-1), b'men!') + def test_uncompressed_interleaved_seek_read(self): + # gh-127847: Make sure the position in the archive is correct + # in the special case of seeking in a ZIP_STORED entry. + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("a.txt", "123") + zipf.writestr("b.txt", "456") + with zipfile.ZipFile(TESTFN, "r") as zipf: + with zipf.open("a.txt", "r") as a, zipf.open("b.txt", "r") as b: + self.assertEqual(a.read(1), b"1") + self.assertEqual(b.seek(1), 1) + self.assertEqual(b.read(1), b"5") + @requires_bz2() def test_decompress_without_3rd_party_library(self): data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 6907ae6d5b7464..f4d396abb6e639 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -819,7 +819,10 @@ def seek(self, offset, whence=0): raise ValueError("Can't reposition in the ZIP file while " "there is an open writing handle on it. " "Close the writing handle before trying to read.") - self._file.seek(offset, whence) + if whence == os.SEEK_CUR: + self._file.seek(self._pos + offset) + else: + self._file.seek(offset, whence) self._pos = self._file.tell() return self._pos diff --git a/Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst b/Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst new file mode 100644 index 00000000000000..3d6e36fb538bca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-12-07-27-51.gh-issue-127847.ksfNKM.rst @@ -0,0 +1 @@ +Fix the position when doing interleaved seeks and reads in uncompressed, unencrypted zip files returned by :meth:`zipfile.ZipFile.open`. From 7985d460c731b2c48419a33fc1820f9512bb6f21 Mon Sep 17 00:00:00 2001 From: Bogdan Romanyuk <65823030+wrongnull@users.noreply.github.com> Date: Tue, 24 Dec 2024 21:00:24 +0300 Subject: [PATCH 282/427] gh-128227: Regenerate `Doc/requirements-oldest-sphinx.txt` (#128228) --- Doc/requirements-oldest-sphinx.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt index 3483faea6b56cb..c8027a05706c21 100644 --- a/Doc/requirements-oldest-sphinx.txt +++ b/Doc/requirements-oldest-sphinx.txt @@ -13,15 +13,15 @@ python-docs-theme>=2022.1 # Sphinx 7.2.6 comes from ``needs_sphinx = '7.2.6'`` in ``Doc/conf.py``. alabaster==0.7.16 -Babel==2.16.0 -certifi==2024.8.30 +babel==2.16.0 +certifi==2024.12.14 charset-normalizer==3.4.0 docutils==0.20.1 idna==3.10 imagesize==1.4.1 -Jinja2==3.1.4 -MarkupSafe==3.0.1 -packaging==24.1 +Jinja2==3.1.5 +MarkupSafe==3.0.2 +packaging==24.2 Pygments==2.18.0 requests==2.32.3 snowballstemmer==2.2.0 @@ -32,4 +32,4 @@ sphinxcontrib-htmlhelp==2.1.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 -urllib3==2.2.3 +urllib3==2.3.0 From 418114c139666f33abff937e40ccbbbdce15bc39 Mon Sep 17 00:00:00 2001 From: Will Childs-Klein Date: Tue, 24 Dec 2024 12:29:27 -0600 Subject: [PATCH 283/427] gh-128035: Add ssl.HAS_PHA to detect libssl PHA support (GH-128036) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add ssl.HAS_PHA to detect libssl Post-Handshake-Auth support Co-authored-by: Tomas R. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/ssl.rst | 6 ++++++ Doc/whatsnew/3.14.rst | 8 ++++++++ Lib/ssl.py | 2 +- Lib/test/test_httplib.py | 4 ++-- Lib/test/test_ssl.py | 3 ++- .../2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst | 1 + Modules/_ssl.c | 6 ++++++ 7 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index f07d151a885692..9d7b6aa66cd443 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -934,6 +934,12 @@ Constants .. versionadded:: 3.13 +.. data:: HAS_PHA + + Whether the OpenSSL library has built-in support for TLS-PHA. + + .. versionadded:: next + .. data:: CHANNEL_BINDING_TYPES List of supported TLS channel binding types. Strings in this list diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 97a37a82f76b9b..0dcee56b7d233f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -584,6 +584,14 @@ pydoc (Contributed by Jelle Zijlstra in :gh:`101552`.) +ssl +--- + +* Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports + TLSv1.3 post-handshake client authentication (PHA). + (Contributed by Will Childs-Klein in :gh:`128036`.) + + symtable -------- diff --git a/Lib/ssl.py b/Lib/ssl.py index c8703b046cfd4b..05df4ad7f0f05c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -116,7 +116,7 @@ from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK, HAS_PHA ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 9d853d254db7c6..89963dadeb152b 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2073,8 +2073,8 @@ def test_host_port(self): def test_tls13_pha(self): import ssl - if not ssl.HAS_TLSv1_3: - self.skipTest('TLS 1.3 support required') + if not ssl.HAS_TLSv1_3 or not ssl.HAS_PHA: + self.skipTest('TLS 1.3 PHA support required') # just check status of PHA flag h = client.HTTPSConnection('localhost', 443) self.assertTrue(h._context.post_handshake_auth) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 3f6f890bbdc658..c16ef3f96f9a21 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4494,7 +4494,8 @@ def server_callback(identity): s.connect((HOST, server.port)) -@unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") +@unittest.skipUnless(has_tls_version('TLSv1_3') and ssl.HAS_PHA, + "Test needs TLS 1.3 PHA") class TestPostHandshakeAuth(unittest.TestCase): def test_pha_setter(self): protocols = [ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst new file mode 100644 index 00000000000000..27815d48425334 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-18-20-37.gh-issue-128035.JwqHdB.rst @@ -0,0 +1 @@ +Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports TLSv1.3 post-handshake client authentication (PHA). Patch by Will Childs-Klein. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index e7df132869fee6..74cf99957389e2 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6553,6 +6553,12 @@ sslmodule_init_constants(PyObject *m) addbool(m, "HAS_PSK", 1); #endif +#ifdef SSL_VERIFY_POST_HANDSHAKE + addbool(m, "HAS_PHA", 1); +#else + addbool(m, "HAS_PHA", 0); +#endif + #undef addbool #undef ADD_INT_CONST From d9ed42bc00c74b3150be2a0eb28da03e01dffcc7 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 25 Dec 2024 09:26:51 +0200 Subject: [PATCH 284/427] gh-128201: Fix ``DeprecationWarning`` in ``test_pdb`` (#128202) --- Lib/test/test_pdb.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 58295cff84310f..9b0806d8b2a9bd 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2065,10 +2065,7 @@ def test_pdb_next_command_for_coroutine(): ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) + ... asyncio.run(test_main()) ... print("finished") >>> with PdbTestInput(['step', @@ -2129,10 +2126,7 @@ def test_pdb_next_command_for_asyncgen(): ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio._set_event_loop_policy(None) + ... asyncio.run(test_main()) ... print("finished") >>> with PdbTestInput(['step', @@ -2250,10 +2244,7 @@ def test_pdb_return_command_for_coroutine(): ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio._set_event_loop_policy(None) + ... asyncio.run(test_main()) ... print("finished") >>> with PdbTestInput(['step', @@ -2350,10 +2341,7 @@ def test_pdb_until_command_for_coroutine(): ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio._set_event_loop_policy(None) + ... asyncio.run(test_main()) ... print("finished") >>> with PdbTestInput(['step', From 76f1785657fde0ebee082b0df54da8f7c911c369 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 25 Dec 2024 17:51:27 +0530 Subject: [PATCH 285/427] gh-128002: use internal llist implementation for asyncio tasks (#128256) --- Modules/_asynciomodule.c | 70 ++++++++++++---------------------------- 1 file changed, 20 insertions(+), 50 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 27c16364457336..603b77a70b15d4 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -6,9 +6,9 @@ #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION_MUT() #include "pycore_dict.h" // _PyDict_GetItem_KnownHash() #include "pycore_freelist.h" // _Py_FREELIST_POP() +#include "pycore_llist.h" // struct llist_node #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_object.h" // _Py_SetImmortalUntracked() #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -60,8 +60,7 @@ typedef struct TaskObj { PyObject *task_coro; PyObject *task_name; PyObject *task_context; - struct TaskObj *next; - struct TaskObj *prev; + struct llist_node task_node; } TaskObj; typedef struct { @@ -136,21 +135,11 @@ typedef struct { /* Counter for autogenerated Task names */ uint64_t task_name_counter; - /* Circular linked-list of all tasks which are instances of asyncio.Task or subclasses - of it. Third party tasks implementations which don't inherit from - asyncio.Task are tracked separately using the 'non_asyncio_tasks' WeakSet. - `first` is used as a sentinel to mark the end of the linked-list. It avoids one - branch in checking for empty list when adding a new task, the list is - initialized with `head`, `head->next` and `head->prev` pointing to `first` - to mark an empty list. - + /* Head of circular linked-list of all tasks which are instances of `asyncio.Task` + or subclasses of it. Third party tasks implementations which don't inherit from + `asyncio.Task` are tracked separately using the `non_asyncio_tasks` WeakSet. */ - - struct { - TaskObj first; - TaskObj *head; - } asyncio_tasks; - + struct llist_node asyncio_tasks_head; } asyncio_state; static inline asyncio_state * @@ -1896,19 +1885,12 @@ register_task(asyncio_state *state, TaskObj *task) { ASYNCIO_STATE_LOCK(state); assert(Task_Check(state, task)); - assert(task != &state->asyncio_tasks.first); - if (task->next != NULL) { + if (task->task_node.next != NULL) { // already registered + assert(task->task_node.prev != NULL); goto exit; } - assert(task->prev == NULL); - assert(state->asyncio_tasks.head != NULL); - - task->next = state->asyncio_tasks.head; - task->prev = state->asyncio_tasks.head->prev; - state->asyncio_tasks.head->prev->next = task; - state->asyncio_tasks.head->prev = task; - + llist_insert_tail(&state->asyncio_tasks_head, &task->task_node); exit: ASYNCIO_STATE_UNLOCK(state); } @@ -1924,18 +1906,12 @@ unregister_task(asyncio_state *state, TaskObj *task) { ASYNCIO_STATE_LOCK(state); assert(Task_Check(state, task)); - assert(task != &state->asyncio_tasks.first); - if (task->next == NULL) { + if (task->task_node.next == NULL) { // not registered - assert(task->prev == NULL); - assert(state->asyncio_tasks.head != task); + assert(task->task_node.prev == NULL); goto exit; } - task->next->prev = task->prev; - task->prev->next = task->next; - task->next = NULL; - task->prev = NULL; - assert(state->asyncio_tasks.head != task); + llist_remove(&task->task_node); exit: ASYNCIO_STATE_UNLOCK(state); } @@ -3625,20 +3601,18 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) Py_DECREF(eager_iter); int err = 0; ASYNCIO_STATE_LOCK(state); - TaskObj *first = &state->asyncio_tasks.first; - TaskObj *head = state->asyncio_tasks.head->next; - Py_INCREF(head); - while (head != first) - { - if (add_one_task(state, tasks, (PyObject *)head, loop) < 0) { + struct llist_node *node; + llist_for_each_safe(node, &state->asyncio_tasks_head) { + TaskObj *task = llist_data(node, TaskObj, task_node); + Py_INCREF(task); + if (add_one_task(state, tasks, (PyObject *)task, loop) < 0) { + Py_DECREF(task); Py_DECREF(tasks); Py_DECREF(loop); - Py_DECREF(head); err = 1; break; } - Py_INCREF(head->next); - Py_SETREF(head, head->next); + Py_DECREF(task); } ASYNCIO_STATE_UNLOCK(state); if (err) { @@ -3847,11 +3821,7 @@ module_exec(PyObject *mod) { asyncio_state *state = get_asyncio_state(mod); - Py_SET_TYPE(&state->asyncio_tasks.first, state->TaskType); - _Py_SetImmortalUntracked((PyObject *)&state->asyncio_tasks.first); - state->asyncio_tasks.head = &state->asyncio_tasks.first; - state->asyncio_tasks.head->next = &state->asyncio_tasks.first; - state->asyncio_tasks.head->prev = &state->asyncio_tasks.first; + llist_init(&state->asyncio_tasks_head); #define CREATE_TYPE(m, tp, spec, base) \ do { \ From 81636d3bbd7f126692326bf957707e8a88c91739 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 25 Dec 2024 13:23:44 +0000 Subject: [PATCH 286/427] gh-128234: support emscripten and wasi in async contextlib tests by removing asyncio from contextlib async tests (#95888) Co-authored-by: Kumar Aditya --- Lib/test/test_contextlib_async.py | 94 ++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 88dcdadd5e027d..d496aa611d1068 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -1,4 +1,4 @@ -import asyncio +import functools from contextlib import ( asynccontextmanager, AbstractAsyncContextManager, AsyncExitStack, nullcontext, aclosing, contextmanager) @@ -8,14 +8,31 @@ from test.test_contextlib import TestBaseExitStack -support.requires_working_socket(module=True) -def tearDownModule(): - asyncio._set_event_loop_policy(None) +def _run_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + coro.send(None) + except StopIteration as e: + return e.value + else: + raise AssertionError("coroutine did not complete") + finally: + coro.close() -class TestAbstractAsyncContextManager(unittest.IsolatedAsyncioTestCase): +def _async_test(async_fn): + """Decorator to turn an async function into a synchronous function""" + @functools.wraps(async_fn) + def wrapper(*args, **kwargs): + return _run_async_fn(async_fn, *args, **kwargs) + return wrapper + + +class TestAbstractAsyncContextManager(unittest.TestCase): + + @_async_test async def test_enter(self): class DefaultEnter(AbstractAsyncContextManager): async def __aexit__(self, *args): @@ -27,6 +44,7 @@ async def __aexit__(self, *args): async with manager as context: self.assertIs(manager, context) + @_async_test async def test_slots(self): class DefaultAsyncContextManager(AbstractAsyncContextManager): __slots__ = () @@ -38,6 +56,7 @@ async def __aexit__(self, *args): manager = DefaultAsyncContextManager() manager.var = 42 + @_async_test async def test_async_gen_propagates_generator_exit(self): # A regression test for https://bugs.python.org/issue33786. @@ -88,8 +107,9 @@ class NoneAexit(ManagerFromScratch): self.assertFalse(issubclass(NoneAexit, AbstractAsyncContextManager)) -class AsyncContextManagerTestCase(unittest.IsolatedAsyncioTestCase): +class AsyncContextManagerTestCase(unittest.TestCase): + @_async_test async def test_contextmanager_plain(self): state = [] @asynccontextmanager @@ -103,6 +123,7 @@ async def woohoo(): state.append(x) self.assertEqual(state, [1, 42, 999]) + @_async_test async def test_contextmanager_finally(self): state = [] @asynccontextmanager @@ -120,6 +141,7 @@ async def woohoo(): raise ZeroDivisionError() self.assertEqual(state, [1, 42, 999]) + @_async_test async def test_contextmanager_traceback(self): @asynccontextmanager async def f(): @@ -175,6 +197,7 @@ class StopAsyncIterationSubclass(StopAsyncIteration): self.assertEqual(frames[0].name, 'test_contextmanager_traceback') self.assertEqual(frames[0].line, 'raise stop_exc') + @_async_test async def test_contextmanager_no_reraise(self): @asynccontextmanager async def whee(): @@ -184,6 +207,7 @@ async def whee(): # Calling __aexit__ should not result in an exception self.assertFalse(await ctx.__aexit__(TypeError, TypeError("foo"), None)) + @_async_test async def test_contextmanager_trap_yield_after_throw(self): @asynccontextmanager async def whoo(): @@ -199,6 +223,7 @@ async def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.ag_suspended) + @_async_test async def test_contextmanager_trap_no_yield(self): @asynccontextmanager async def whoo(): @@ -208,6 +233,7 @@ async def whoo(): with self.assertRaises(RuntimeError): await ctx.__aenter__() + @_async_test async def test_contextmanager_trap_second_yield(self): @asynccontextmanager async def whoo(): @@ -221,6 +247,7 @@ async def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.ag_suspended) + @_async_test async def test_contextmanager_non_normalised(self): @asynccontextmanager async def whoo(): @@ -234,6 +261,7 @@ async def whoo(): with self.assertRaises(SyntaxError): await ctx.__aexit__(RuntimeError, None, None) + @_async_test async def test_contextmanager_except(self): state = [] @asynccontextmanager @@ -251,6 +279,7 @@ async def woohoo(): raise ZeroDivisionError(999) self.assertEqual(state, [1, 42, 999]) + @_async_test async def test_contextmanager_except_stopiter(self): @asynccontextmanager async def woohoo(): @@ -277,6 +306,7 @@ class StopAsyncIterationSubclass(StopAsyncIteration): else: self.fail(f'{stop_exc} was suppressed') + @_async_test async def test_contextmanager_wrap_runtimeerror(self): @asynccontextmanager async def woohoo(): @@ -321,12 +351,14 @@ def test_contextmanager_doc_attrib(self): self.assertEqual(baz.__doc__, "Whee!") @support.requires_docstrings + @_async_test async def test_instance_docstring_given_cm_docstring(self): baz = self._create_contextmanager_attribs()(None) self.assertEqual(baz.__doc__, "Whee!") async with baz: pass # suppress warning + @_async_test async def test_keywords(self): # Ensure no keyword arguments are inhibited @asynccontextmanager @@ -335,6 +367,7 @@ async def woohoo(self, func, args, kwds): async with woohoo(self=11, func=22, args=33, kwds=44) as target: self.assertEqual(target, (11, 22, 33, 44)) + @_async_test async def test_recursive(self): depth = 0 ncols = 0 @@ -361,6 +394,7 @@ async def recursive(): self.assertEqual(ncols, 10) self.assertEqual(depth, 0) + @_async_test async def test_decorator(self): entered = False @@ -379,6 +413,7 @@ async def test(): await test() self.assertFalse(entered) + @_async_test async def test_decorator_with_exception(self): entered = False @@ -401,6 +436,7 @@ async def test(): await test() self.assertFalse(entered) + @_async_test async def test_decorating_method(self): @asynccontextmanager @@ -435,7 +471,7 @@ async def method(self, a, b, c=None): self.assertEqual(test.b, 2) -class AclosingTestCase(unittest.IsolatedAsyncioTestCase): +class AclosingTestCase(unittest.TestCase): @support.requires_docstrings def test_instance_docs(self): @@ -443,6 +479,7 @@ def test_instance_docs(self): obj = aclosing(None) self.assertEqual(obj.__doc__, cm_docstring) + @_async_test async def test_aclosing(self): state = [] class C: @@ -454,6 +491,7 @@ async def aclose(self): self.assertEqual(x, y) self.assertEqual(state, [1]) + @_async_test async def test_aclosing_error(self): state = [] class C: @@ -467,6 +505,7 @@ async def aclose(self): 1 / 0 self.assertEqual(state, [1]) + @_async_test async def test_aclosing_bpo41229(self): state = [] @@ -492,45 +531,27 @@ async def agenfunc(): self.assertEqual(state, [1]) -class TestAsyncExitStack(TestBaseExitStack, unittest.IsolatedAsyncioTestCase): +class TestAsyncExitStack(TestBaseExitStack, unittest.TestCase): class SyncAsyncExitStack(AsyncExitStack): - @staticmethod - def run_coroutine(coro): - loop = asyncio.new_event_loop() - t = loop.create_task(coro) - t.add_done_callback(lambda f: loop.stop()) - loop.run_forever() - - exc = t.exception() - if not exc: - return t.result() - else: - context = exc.__context__ - - try: - raise exc - except: - exc.__context__ = context - raise exc def close(self): - return self.run_coroutine(self.aclose()) + return _run_async_fn(self.aclose) def __enter__(self): - return self.run_coroutine(self.__aenter__()) + return _run_async_fn(self.__aenter__) def __exit__(self, *exc_details): - return self.run_coroutine(self.__aexit__(*exc_details)) + return _run_async_fn(self.__aexit__, *exc_details) exit_stack = SyncAsyncExitStack callback_error_internal_frames = [ - ('__exit__', 'return self.run_coroutine(self.__aexit__(*exc_details))'), - ('run_coroutine', 'raise exc'), - ('run_coroutine', 'raise exc'), + ('__exit__', 'return _run_async_fn(self.__aexit__, *exc_details)'), + ('_run_async_fn', 'coro.send(None)'), ('__aexit__', 'raise exc'), ('__aexit__', 'cb_suppress = cb(*exc_details)'), ] + @_async_test async def test_async_callback(self): expected = [ ((), {}), @@ -573,6 +594,7 @@ async def _exit(*args, **kwds): stack.push_async_callback(callback=_exit, arg=3) self.assertEqual(result, []) + @_async_test async def test_async_push(self): exc_raised = ZeroDivisionError async def _expect_exc(exc_type, exc, exc_tb): @@ -608,6 +630,7 @@ async def __aexit__(self, *exc_details): self.assertIs(stack._exit_callbacks[-1][1], _expect_exc) 1/0 + @_async_test async def test_enter_async_context(self): class TestCM(object): async def __aenter__(self): @@ -629,6 +652,7 @@ async def _exit(): self.assertEqual(result, [1, 2, 3, 4]) + @_async_test async def test_enter_async_context_errors(self): class LacksEnterAndExit: pass @@ -648,6 +672,7 @@ async def __aenter__(self): await stack.enter_async_context(LacksExit()) self.assertFalse(stack._exit_callbacks) + @_async_test async def test_async_exit_exception_chaining(self): # Ensure exception chaining matches the reference behaviour async def raise_exc(exc): @@ -679,6 +704,7 @@ async def suppress_exc(*exc_details): self.assertIsInstance(inner_exc, ValueError) self.assertIsInstance(inner_exc.__context__, ZeroDivisionError) + @_async_test async def test_async_exit_exception_explicit_none_context(self): # Ensure AsyncExitStack chaining matches actual nested `with` statements # regarding explicit __context__ = None. @@ -713,6 +739,7 @@ async def my_cm_with_exit_stack(): else: self.fail("Expected IndexError, but no exception was raised") + @_async_test async def test_instance_bypass_async(self): class Example(object): pass cm = Example() @@ -725,7 +752,8 @@ class Example(object): pass self.assertIs(stack._exit_callbacks[-1][1], cm) -class TestAsyncNullcontext(unittest.IsolatedAsyncioTestCase): +class TestAsyncNullcontext(unittest.TestCase): + @_async_test async def test_async_nullcontext(self): class C: pass From 5c814c83cdd3dc42bd9682106ffb7ade7ce6b5b3 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Wed, 25 Dec 2024 18:42:04 +0100 Subject: [PATCH 287/427] gh-128198: Add missing error checks for usages of PyIter_Next() (GH-128199) --- Modules/_asynciomodule.c | 13 +++++++++++++ Objects/frameobject.c | 4 ++++ Objects/namespaceobject.c | 4 ++++ 3 files changed, 21 insertions(+) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 603b77a70b15d4..74db4c74af905a 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3599,6 +3599,13 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) Py_DECREF(item); } Py_DECREF(eager_iter); + + if (PyErr_Occurred()) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + int err = 0; ASYNCIO_STATE_LOCK(state); struct llist_node *node; @@ -3636,6 +3643,12 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) } Py_DECREF(scheduled_iter); Py_DECREF(loop); + + if (PyErr_Occurred()) { + Py_DECREF(tasks); + return NULL; + } + return tasks; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 10fd3a982c36f4..4f0040df4f3017 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -264,6 +264,10 @@ framelocalsproxy_merge(PyObject* self, PyObject* other) Py_DECREF(iter); + if (PyErr_Occurred()) { + return -1; + } + return 0; } diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index 5b7547103a2b3f..4ef3bd92f5a569 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -141,6 +141,10 @@ namespace_repr(PyObject *ns) goto error; } + if (PyErr_Occurred()) { + goto error; + } + separator = PyUnicode_FromString(", "); if (separator == NULL) goto error; From 3eb746a7b9122c7c3cfc3fbd41d9d079d8b02845 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 26 Dec 2024 20:02:23 +0530 Subject: [PATCH 288/427] gh-127949: add docs for asyncio policy deprecation (#128269) --- Doc/library/asyncio-policy.rst | 8 ++++++++ Doc/whatsnew/3.14.rst | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index ea7fe957da7c40..9f86234ce941d1 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -7,6 +7,14 @@ Policies ======== +.. warning:: + + Policies are deprecated and will be removed in Python 3.16. + Users are encouraged to use the :func:`asyncio.run` function + or the :class:`asyncio.Runner` with *loop_factory* to use + the desired loop implementation. + + An event loop policy is a global object used to get and set the current :ref:`event loop `, as well as create new event loops. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 0dcee56b7d233f..935c61c474e889 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -691,10 +691,36 @@ Deprecated (Contributed by Serhiy Storchaka in :gh:`58032`.) * :mod:`asyncio`: - :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16, - use :func:`inspect.iscoroutinefunction` instead. - (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + + * :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16; + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + + * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. + In particular, the following classes and functions are deprecated: + + * :class:`asyncio.AbstractEventLoopPolicy` + * :class:`asyncio.DefaultEventLoopPolicy` + * :class:`asyncio.WindowsSelectorEventLoopPolicy` + * :class:`asyncio.WindowsProactorEventLoopPolicy` + * :func:`asyncio.get_event_loop_policy` + * :func:`asyncio.set_event_loop_policy` + * :func:`asyncio.set_event_loop` + + Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with + *loop_factory* to use the desired event loop implementation. + + For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + + import asyncio + + async def main(): + ... + + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + + (Contributed by Kumar Aditya in :gh:`127949`.) * :mod:`builtins`: Passing a complex number as the *real* or *imag* argument in the From c6563f3f22e715780e8182481879b0c9110b71ac Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Thu, 26 Dec 2024 15:39:15 +0100 Subject: [PATCH 289/427] gh-119786: Fix typos in `InternalDocs/frames.md` (#128275) Fix typos in `InternalDocs/frames.md` --- InternalDocs/frames.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index 2598873ca98479..1a909009eea610 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -33,7 +33,7 @@ Each activation record is laid out as: * Stack This seems to provide the best performance without excessive complexity. -The specials have a fixed size, so the offset of the locals is know. The +The specials have a fixed size, so the offset of the locals is known. The interpreter needs to hold two pointers, a frame pointer and a stack pointer. #### Alternative layout @@ -52,7 +52,7 @@ an extra pointer for the locals, which can hurt performance. ### Generators and Coroutines Generators and coroutines contain a `_PyInterpreterFrame` -The specials sections contains the following pointers: +The specials section contains the following pointers: * Globals dict * Builtins dict @@ -69,7 +69,7 @@ and builtins, than strong references to both globals and builtins. When creating a backtrace or when calling `sys._getframe()` the frame becomes visible to Python code. When this happens a new `PyFrameObject` is created -and a strong reference to it placed in the `frame_obj` field of the specials +and a strong reference to it is placed in the `frame_obj` field of the specials section. The `frame_obj` field is initially `NULL`. The `PyFrameObject` may outlive a stack-allocated `_PyInterpreterFrame`. @@ -128,7 +128,7 @@ The `return_offset` field determines where a `RETURN` should go in the caller, relative to `instr_ptr`. It is only meaningful to the callee, so it needs to be set in any instruction that implements a call (to a Python function), including CALL, SEND and BINARY_SUBSCR_GETITEM, among others. If there is no -callee, then return_offset is meaningless. It is necessary to have a separate +callee, then return_offset is meaningless. It is necessary to have a separate field for the return offset because (1) if we apply this offset to `instr_ptr` while executing the `RETURN`, this is too early and would lose us information about the previous instruction which we could need for introspecting and From 8a26c7b2af59b999ae41b8189295cc550f297f6e Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Thu, 26 Dec 2024 23:39:44 +0900 Subject: [PATCH 290/427] Docs: Fix comment out in `c-api/typeobj.rst` (#128266) --- Doc/c-api/typeobj.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index ba58cc1c26c70b..69c518d8e64cbc 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1023,6 +1023,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have ``NULL`` values. + .. XXX are most flag bits *really* inherited individually? **Default:** From 42f7a00ae8b6b3fa09115e24b9512216c6c8978e Mon Sep 17 00:00:00 2001 From: da-woods Date: Thu, 26 Dec 2024 14:40:48 +0000 Subject: [PATCH 291/427] Clean up redundant ifdef in list getitem (#128257) It's already inside a `Py_GIL_DISABLED` block so the `#else` clause is always unused. --- Objects/listobject.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index a877bad66be45f..bbd53e7de94a31 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -335,11 +335,7 @@ list_item_impl(PyListObject *self, Py_ssize_t idx) if (!valid_index(idx, size)) { goto exit; } -#ifdef Py_GIL_DISABLED item = _Py_NewRefWithLock(self->ob_item[idx]); -#else - item = Py_NewRef(self->ob_item[idx]); -#endif exit: Py_END_CRITICAL_SECTION(); return item; From 9ddc388527477afdae17252628d63138631072ba Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 26 Dec 2024 14:50:20 +0000 Subject: [PATCH 292/427] gh-124761: add `socket.SO_REUSEPORT_LB` (#124961) --- Doc/library/socket.rst | 8 ++++++++ .../2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst | 1 + Modules/socketmodule.c | 3 +++ 3 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 58323ba6514eac..8ba2bd1dcce8cc 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -674,6 +674,14 @@ Constants .. availability:: Linux >= 3.9 +.. data:: SO_REUSEPORT_LB + + Constant to enable duplicate address and port bindings with load balancing. + + .. versionadded:: next + + .. availability:: FreeBSD >= 12.0 + .. data:: AF_HYPERV HV_PROTOCOL_RAW HVSOCKET_CONNECT_TIMEOUT diff --git a/Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst b/Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst new file mode 100644 index 00000000000000..797dd31b368548 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-04-09-56-45.gh-issue-124761.N4pSD6.rst @@ -0,0 +1 @@ +Add :data:`~socket.SO_REUSEPORT_LB` constant to :mod:`socket` for FreeBSD. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 9394f1c940bedf..1e95be9b1bc9f4 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7916,6 +7916,9 @@ socket_exec(PyObject *m) ADD_INT_MACRO(m, SO_REUSEPORT); #endif #endif +#ifdef SO_REUSEPORT_LB + ADD_INT_MACRO(m, SO_REUSEPORT_LB); +#endif #ifdef SO_SNDBUF ADD_INT_MACRO(m, SO_SNDBUF); #endif From fb0b94223d481ca58b7c77332348d1ab2c9ab272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 26 Dec 2024 16:03:57 +0100 Subject: [PATCH 293/427] gh-87138: convert blake2b/2s types to heap types (#127669) --- Modules/blake2module.c | 89 ++++++++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 94cdfe7fd2e962..6723e7de4675a5 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -379,13 +379,13 @@ class _blake2.blake2s "Blake2Object *" "&PyBlake2_BLAKE2sType" static Blake2Object * new_Blake2Object(PyTypeObject *type) { - Blake2Object *self; - self = (Blake2Object *)type->tp_alloc(type, 0); + Blake2Object *self = PyObject_GC_New(Blake2Object, type); if (self == NULL) { return NULL; } HASHLIB_INIT_MUTEX(self); + PyObject_GC_Track(self); return self; } @@ -454,7 +454,28 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, } self->impl = type_to_impl(type); - + // Ensure that the states are NULL-initialized in case of an error. + // See: py_blake2_clear() for more details. + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + self->blake2b_256_state = NULL; + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + self->blake2s_128_state = NULL; + break; +#endif + case Blake2b: + self->blake2b_state = NULL; + break; + case Blake2s: + self->blake2s_state = NULL; + break; + default: + Py_UNREACHABLE(); + } // Using Blake2b because we statically know that these are greater than the // Blake2s sizes -- this avoids a VLA. uint8_t salt_[HACL_HASH_BLAKE2B_SALT_BYTES] = { 0 }; @@ -595,7 +616,7 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, return (PyObject *)self; error: - Py_XDECREF(self); + Py_XDECREF(self); return NULL; } @@ -875,46 +896,70 @@ static PyGetSetDef py_blake2b_getsetters[] = { {NULL} }; - -static void -py_blake2b_dealloc(Blake2Object *self) +static int +py_blake2_clear(PyObject *op) { + Blake2Object *self = (Blake2Object *)op; + // The initialization function uses PyObject_GC_New() but explicitly + // initializes the HACL* internal state to NULL before allocating + // it. If an error occurs in the constructor, we should only free + // states that were allocated (i.e. that are not NULL). switch (self->impl) { #if HACL_CAN_COMPILE_SIMD256 case Blake2b_256: - if (self->blake2b_256_state != NULL) + if (self->blake2b_256_state != NULL) { Hacl_Hash_Blake2b_Simd256_free(self->blake2b_256_state); + self->blake2b_256_state = NULL; + } break; #endif #if HACL_CAN_COMPILE_SIMD128 case Blake2s_128: - if (self->blake2s_128_state != NULL) + if (self->blake2s_128_state != NULL) { Hacl_Hash_Blake2s_Simd128_free(self->blake2s_128_state); + self->blake2s_128_state = NULL; + } break; #endif case Blake2b: - // This happens if we hit "goto error" in the middle of the - // initialization function. We leverage the fact that tp_alloc - // guarantees that the contents of the object are NULL-initialized - // (see documentation for PyType_GenericAlloc) to detect this case. - if (self->blake2b_state != NULL) + if (self->blake2b_state != NULL) { Hacl_Hash_Blake2b_free(self->blake2b_state); + self->blake2b_state = NULL; + } break; case Blake2s: - if (self->blake2s_state != NULL) + if (self->blake2s_state != NULL) { Hacl_Hash_Blake2s_free(self->blake2s_state); + self->blake2s_state = NULL; + } break; default: Py_UNREACHABLE(); } + return 0; +} +static void +py_blake2_dealloc(PyObject *self) +{ PyTypeObject *type = Py_TYPE(self); - PyObject_Free(self); + PyObject_GC_UnTrack(self); + (void)py_blake2_clear(self); + type->tp_free(self); Py_DECREF(type); } +static int +py_blake2_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + static PyType_Slot blake2b_type_slots[] = { - {Py_tp_dealloc, py_blake2b_dealloc}, + {Py_tp_clear, py_blake2_clear}, + {Py_tp_dealloc, py_blake2_dealloc}, + {Py_tp_traverse, py_blake2_traverse}, {Py_tp_doc, (char *)py_blake2b_new__doc__}, {Py_tp_methods, py_blake2b_methods}, {Py_tp_getset, py_blake2b_getsetters}, @@ -923,7 +968,9 @@ static PyType_Slot blake2b_type_slots[] = { }; static PyType_Slot blake2s_type_slots[] = { - {Py_tp_dealloc, py_blake2b_dealloc}, + {Py_tp_clear, py_blake2_clear}, + {Py_tp_dealloc, py_blake2_dealloc}, + {Py_tp_traverse, py_blake2_traverse}, {Py_tp_doc, (char *)py_blake2s_new__doc__}, {Py_tp_methods, py_blake2b_methods}, {Py_tp_getset, py_blake2b_getsetters}, @@ -936,13 +983,15 @@ static PyType_Slot blake2s_type_slots[] = { static PyType_Spec blake2b_type_spec = { .name = "_blake2.blake2b", .basicsize = sizeof(Blake2Object), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE, .slots = blake2b_type_slots }; static PyType_Spec blake2s_type_spec = { .name = "_blake2.blake2s", .basicsize = sizeof(Blake2Object), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE, .slots = blake2s_type_slots }; From 3bd7730bbda3e38db5920a7b8c95958ca90342bf Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 26 Dec 2024 16:17:22 +0100 Subject: [PATCH 294/427] gh-126868: Add freelist for compact ints to `_PyLong_New` (#128181) Co-authored-by: Kumar Aditya --- ...-12-22-15-47-44.gh-issue-126868.RpjKez.rst | 1 + Objects/longobject.c | 30 +++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst new file mode 100644 index 00000000000000..ede383deb4ad31 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-22-15-47-44.gh-issue-126868.RpjKez.rst @@ -0,0 +1 @@ +Increase usage of freelist for :class:`int` allocation. diff --git a/Objects/longobject.c b/Objects/longobject.c index bd7ff68d0899c6..d449a01cedf886 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -156,7 +156,7 @@ PyLongObject * _PyLong_New(Py_ssize_t size) { assert(size >= 0); - PyLongObject *result; + PyLongObject *result = NULL; if (size > (Py_ssize_t)MAX_LONG_DIGITS) { PyErr_SetString(PyExc_OverflowError, "too many digits in integer"); @@ -165,19 +165,25 @@ _PyLong_New(Py_ssize_t size) /* Fast operations for single digit integers (including zero) * assume that there is always at least one digit present. */ Py_ssize_t ndigits = size ? size : 1; - /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + - sizeof(digit)*size. Previous incarnations of this code used - sizeof() instead of the offsetof, but this risks being - incorrect in the presence of padding between the header - and the digits. */ - result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) + - ndigits*sizeof(digit)); - if (!result) { - PyErr_NoMemory(); - return NULL; + + if (ndigits == 1) { + result = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); + } + if (result == NULL) { + /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + + sizeof(digit)*size. Previous incarnations of this code used + sizeof() instead of the offsetof, but this risks being + incorrect in the presence of padding between the header + and the digits. */ + result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) + + ndigits*sizeof(digit)); + if (!result) { + PyErr_NoMemory(); + return NULL; + } + _PyObject_Init((PyObject*)result, &PyLong_Type); } _PyLong_SetSignAndDigitCount(result, size != 0, size); - _PyObject_Init((PyObject*)result, &PyLong_Type); /* The digit has to be initialized explicitly to avoid * use-of-uninitialized-value. */ result->long_value.ob_digit[0] = 0; From ea2b53739f1128184b4140decbeffeac6cfe966f Mon Sep 17 00:00:00 2001 From: Moshe Kaplan Date: Thu, 26 Dec 2024 16:53:37 -0500 Subject: [PATCH 295/427] Remove incorrect imports rationale comment in `http.server` (#128278) Remove reference to gethostbyaddr(), because it's not actually used within this code. --- Lib/http/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/http/server.py b/Lib/http/server.py index a6f7aecc78763f..a90c8d34c394db 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -99,7 +99,7 @@ import posixpath import select import shutil -import socket # For gethostbyaddr() +import socket import socketserver import sys import time From 401bba6b58497ce59e7b45ad33e43ae8c67abcb9 Mon Sep 17 00:00:00 2001 From: CF Bolz-Tereick Date: Fri, 27 Dec 2024 02:03:47 +0100 Subject: [PATCH 296/427] gh-127537: Add __class_getitem__ to the python implementation of functools.partial (#127537) --- Lib/functools.py | 3 +++ Lib/test/test_functools.py | 6 ++++++ .../Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst | 5 +++++ 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst diff --git a/Lib/functools.py b/Lib/functools.py index eff6540c7f606e..786b8aedfd77f5 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -433,6 +433,9 @@ def __setstate__(self, state): self._phcount = phcount self._merger = merger + __class_getitem__ = classmethod(GenericAlias) + + try: from _functools import partial, Placeholder, _PlaceholderType except ImportError: diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index ffd2adb8665b45..4a0252cb637a52 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -473,6 +473,12 @@ class A: self.assertEqual(a.cmeth(3, b=4), ((1, A, 3), {'a': 2, 'b': 4})) self.assertEqual(a.smeth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + def test_partial_genericalias(self): + alias = self.partial[int] + self.assertIs(alias.__origin__, self.partial) + self.assertEqual(alias.__args__, (int,)) + self.assertEqual(alias.__parameters__, ()) + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst b/Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst new file mode 100644 index 00000000000000..5e39933047993c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-04-10-39-29.gh-issue-83662.CG1s3m.rst @@ -0,0 +1,5 @@ +Add missing ``__class_getitem__`` method to the Python implementation of +:func:`functools.partial`, to make it compatible with the C version. This is +mainly relevant for alternative Python implementations like PyPy and +GraalPy, because CPython will usually use the C-implementation of that +function. From 08a0728d6c32986d35edb26872b4512a71ae60f3 Mon Sep 17 00:00:00 2001 From: Damien <81557462+Damien-Chen@users.noreply.github.com> Date: Fri, 27 Dec 2024 09:57:55 +0800 Subject: [PATCH 297/427] gh-125887: Update PyObject_HasAttr exception behavior (#125907) Update PyObject_HasAttr exception behavior Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/c-api/object.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index f97ade01e67850..a137688fe07545 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -111,7 +111,8 @@ Object Protocol .. note:: Exceptions that occur when this calls :meth:`~object.__getattr__` and - :meth:`~object.__getattribute__` methods are silently ignored. + :meth:`~object.__getattribute__` methods aren't propagated, + but instead given to :func:`sys.unraisablehook`. For proper error handling, use :c:func:`PyObject_HasAttrWithError`, :c:func:`PyObject_GetOptionalAttr` or :c:func:`PyObject_GetAttr` instead. From 0b5f1fae573a2c658eb000433ad7b87e9c40c697 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 27 Dec 2024 14:58:35 +0100 Subject: [PATCH 298/427] Mention loop_factory argument in docstring for asyncio.run() (#128288) --- Lib/asyncio/runners.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index b9adf291d4817f..14397b4ad0c732 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -177,6 +177,7 @@ def run(main, *, debug=None, loop_factory=None): running in the same thread. If debug is True, the event loop will be run in debug mode. + If loop_factory is passed, it is used for new event loop creation. This function always creates a new event loop and closes it at the end. It should be used as a main entry point for asyncio programs, and should From 71de839ec987bb67b98fcfecfc687281841a713c Mon Sep 17 00:00:00 2001 From: donBarbos Date: Fri, 27 Dec 2024 18:12:25 +0400 Subject: [PATCH 299/427] gh-127089: Add missing description for codes in `http.HTTPStatus` (#127100) Co-authored-by: Ethan Furman Co-authored-by: Andrew Svetlov --- Lib/http/__init__.py | 41 +++++++++++++++++++++++++--------------- Lib/test/test_httplib.py | 41 +++++++++++++++++++++++++--------------- 2 files changed, 52 insertions(+), 30 deletions(-) diff --git a/Lib/http/__init__.py b/Lib/http/__init__.py index d64741ec0dd29a..9f278289420713 100644 --- a/Lib/http/__init__.py +++ b/Lib/http/__init__.py @@ -54,8 +54,9 @@ def is_server_error(self): CONTINUE = 100, 'Continue', 'Request received, please continue' SWITCHING_PROTOCOLS = (101, 'Switching Protocols', 'Switching to new protocol; obey Upgrade header') - PROCESSING = 102, 'Processing' - EARLY_HINTS = 103, 'Early Hints' + PROCESSING = 102, 'Processing', 'Server is processing the request' + EARLY_HINTS = (103, 'Early Hints', + 'Headers sent to prepare for the response') # success OK = 200, 'OK', 'Request fulfilled, document follows' @@ -67,9 +68,11 @@ def is_server_error(self): NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows' RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input' PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows' - MULTI_STATUS = 207, 'Multi-Status' - ALREADY_REPORTED = 208, 'Already Reported' - IM_USED = 226, 'IM Used' + MULTI_STATUS = (207, 'Multi-Status', + 'Response contains multiple statuses in the body') + ALREADY_REPORTED = (208, 'Already Reported', + 'Operation has already been reported') + IM_USED = 226, 'IM Used', 'Request completed using instance manipulations' # redirection MULTIPLE_CHOICES = (300, 'Multiple Choices', @@ -128,15 +131,19 @@ def is_server_error(self): EXPECTATION_FAILED = (417, 'Expectation Failed', 'Expect condition could not be satisfied') IM_A_TEAPOT = (418, 'I\'m a Teapot', - 'Server refuses to brew coffee because it is a teapot.') + 'Server refuses to brew coffee because it is a teapot') MISDIRECTED_REQUEST = (421, 'Misdirected Request', 'Server is not able to produce a response') - UNPROCESSABLE_CONTENT = 422, 'Unprocessable Content' + UNPROCESSABLE_CONTENT = (422, 'Unprocessable Content', + 'Server is not able to process the contained instructions') UNPROCESSABLE_ENTITY = UNPROCESSABLE_CONTENT - LOCKED = 423, 'Locked' - FAILED_DEPENDENCY = 424, 'Failed Dependency' - TOO_EARLY = 425, 'Too Early' - UPGRADE_REQUIRED = 426, 'Upgrade Required' + LOCKED = 423, 'Locked', 'Resource of a method is locked' + FAILED_DEPENDENCY = (424, 'Failed Dependency', + 'Dependent action of the request failed') + TOO_EARLY = (425, 'Too Early', + 'Server refuses to process a request that might be replayed') + UPGRADE_REQUIRED = (426, 'Upgrade Required', + 'Server refuses to perform the request using the current protocol') PRECONDITION_REQUIRED = (428, 'Precondition Required', 'The origin server requires the request to be conditional') TOO_MANY_REQUESTS = (429, 'Too Many Requests', @@ -164,10 +171,14 @@ def is_server_error(self): 'The gateway server did not receive a timely response') HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported', 'Cannot fulfill request') - VARIANT_ALSO_NEGOTIATES = 506, 'Variant Also Negotiates' - INSUFFICIENT_STORAGE = 507, 'Insufficient Storage' - LOOP_DETECTED = 508, 'Loop Detected' - NOT_EXTENDED = 510, 'Not Extended' + VARIANT_ALSO_NEGOTIATES = (506, 'Variant Also Negotiates', + 'Server has an internal configuration error') + INSUFFICIENT_STORAGE = (507, 'Insufficient Storage', + 'Server is not able to store the representation') + LOOP_DETECTED = (508, 'Loop Detected', + 'Server encountered an infinite loop while processing a request') + NOT_EXTENDED = (510, 'Not Extended', + 'Request does not meet the resource access policy') NETWORK_AUTHENTICATION_REQUIRED = (511, 'Network Authentication Required', 'The client needs to authenticate to gain network access') diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 89963dadeb152b..7a7ec555a2dbbb 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -594,8 +594,9 @@ def is_server_error(self): CONTINUE = 100, 'Continue', 'Request received, please continue' SWITCHING_PROTOCOLS = (101, 'Switching Protocols', 'Switching to new protocol; obey Upgrade header') - PROCESSING = 102, 'Processing' - EARLY_HINTS = 103, 'Early Hints' + PROCESSING = 102, 'Processing', 'Server is processing the request' + EARLY_HINTS = (103, 'Early Hints', + 'Headers sent to prepare for the response') # success OK = 200, 'OK', 'Request fulfilled, document follows' CREATED = 201, 'Created', 'Document created, URL follows' @@ -606,9 +607,11 @@ def is_server_error(self): NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows' RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input' PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows' - MULTI_STATUS = 207, 'Multi-Status' - ALREADY_REPORTED = 208, 'Already Reported' - IM_USED = 226, 'IM Used' + MULTI_STATUS = (207, 'Multi-Status', + 'Response contains multiple statuses in the body') + ALREADY_REPORTED = (208, 'Already Reported', + 'Operation has already been reported') + IM_USED = 226, 'IM Used', 'Request completed using instance manipulations' # redirection MULTIPLE_CHOICES = (300, 'Multiple Choices', 'Object has several resources -- see URI list') @@ -665,15 +668,19 @@ def is_server_error(self): EXPECTATION_FAILED = (417, 'Expectation Failed', 'Expect condition could not be satisfied') IM_A_TEAPOT = (418, 'I\'m a Teapot', - 'Server refuses to brew coffee because it is a teapot.') + 'Server refuses to brew coffee because it is a teapot') MISDIRECTED_REQUEST = (421, 'Misdirected Request', 'Server is not able to produce a response') - UNPROCESSABLE_CONTENT = 422, 'Unprocessable Content' + UNPROCESSABLE_CONTENT = (422, 'Unprocessable Content', + 'Server is not able to process the contained instructions') UNPROCESSABLE_ENTITY = UNPROCESSABLE_CONTENT - LOCKED = 423, 'Locked' - FAILED_DEPENDENCY = 424, 'Failed Dependency' - TOO_EARLY = 425, 'Too Early' - UPGRADE_REQUIRED = 426, 'Upgrade Required' + LOCKED = 423, 'Locked', 'Resource of a method is locked' + FAILED_DEPENDENCY = (424, 'Failed Dependency', + 'Dependent action of the request failed') + TOO_EARLY = (425, 'Too Early', + 'Server refuses to process a request that might be replayed') + UPGRADE_REQUIRED = (426, 'Upgrade Required', + 'Server refuses to perform the request using the current protocol') PRECONDITION_REQUIRED = (428, 'Precondition Required', 'The origin server requires the request to be conditional') TOO_MANY_REQUESTS = (429, 'Too Many Requests', @@ -700,10 +707,14 @@ def is_server_error(self): 'The gateway server did not receive a timely response') HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported', 'Cannot fulfill request') - VARIANT_ALSO_NEGOTIATES = 506, 'Variant Also Negotiates' - INSUFFICIENT_STORAGE = 507, 'Insufficient Storage' - LOOP_DETECTED = 508, 'Loop Detected' - NOT_EXTENDED = 510, 'Not Extended' + VARIANT_ALSO_NEGOTIATES = (506, 'Variant Also Negotiates', + 'Server has an internal configuration error') + INSUFFICIENT_STORAGE = (507, 'Insufficient Storage', + 'Server is not able to store the representation') + LOOP_DETECTED = (508, 'Loop Detected', + 'Server encountered an infinite loop while processing a request') + NOT_EXTENDED = (510, 'Not Extended', + 'Request does not meet the resource access policy') NETWORK_AUTHENTICATION_REQUIRED = (511, 'Network Authentication Required', 'The client needs to authenticate to gain network access') From 64173cd6f2d8dc95c6f8b67912d0edd1c1b707d5 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 27 Dec 2024 20:43:41 +0530 Subject: [PATCH 300/427] gh-127949: make deprecation of policy system more prominent (#128290) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/deprecations/pending-removal-in-3.16.rst | 27 +++++++++++++++++++- Doc/library/asyncio-eventloop.rst | 16 ++++++++---- Doc/library/asyncio-runner.rst | 6 +++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index 6f6954b783a1ae..f2b818f14c63cd 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -19,10 +19,35 @@ Pending removal in Python 3.16 * :mod:`asyncio`: * :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16, + and will be removed in Python 3.16; use :func:`inspect.iscoroutinefunction` instead. (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. + In particular, the following classes and functions are deprecated: + + * :class:`asyncio.AbstractEventLoopPolicy` + * :class:`asyncio.DefaultEventLoopPolicy` + * :class:`asyncio.WindowsSelectorEventLoopPolicy` + * :class:`asyncio.WindowsProactorEventLoopPolicy` + * :func:`asyncio.get_event_loop_policy` + * :func:`asyncio.set_event_loop_policy` + * :func:`asyncio.set_event_loop` + + Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with + *loop_factory* to use the desired event loop implementation. + + For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + + import asyncio + + async def main(): + ... + + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + + (Contributed by Kumar Aditya in :gh:`127949`.) + * :mod:`builtins`: * Bitwise inversion on boolean types, ``~True`` or ``~False`` diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 29f843123f8560..ccb362d8c31ddf 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -62,6 +62,13 @@ an event loop: .. versionchanged:: 3.14 Raises a :exc:`RuntimeError` if there is no current event loop. + .. note:: + + The :mod:`!asyncio` policy system is deprecated and will be removed + in Python 3.16; from there on, this function will always return the + running event loop. + + .. function:: set_event_loop(loop) Set *loop* as the current event loop for the current OS thread. @@ -1781,12 +1788,11 @@ By default asyncio is configured to use :class:`EventLoop`. import asyncio import selectors - class MyPolicy(asyncio.DefaultEventLoopPolicy): - def new_event_loop(self): - selector = selectors.SelectSelector() - return asyncio.SelectorEventLoop(selector) + async def main(): + ... - asyncio.set_event_loop_policy(MyPolicy()) + loop_factory = lambda: asyncio.SelectorEventLoop(selectors.SelectSelector()) + asyncio.run(main(), loop_factory=loop_factory) .. availability:: Unix, Windows. diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 28d5aaf3692baa..48d78099fd3ce7 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -76,6 +76,12 @@ Running an asyncio Program *coro* can be any awaitable object. + .. note:: + + The :mod:`!asyncio` policy system is deprecated and will be removed + in Python 3.16; from there on, an explicit *loop_factory* is needed + to configure the event loop. + Runner context manager ====================== From aeb9b65aa26444529e4adc7d6e5b0d3dd9889ec2 Mon Sep 17 00:00:00 2001 From: Stephen Hansen Date: Fri, 27 Dec 2024 17:09:01 -0500 Subject: [PATCH 301/427] gh-127586: multiprocessing.Pool does not properly restore blocked signals (try 2) (GH-128011) Correct pthread_sigmask in resource_tracker to restore old signals Using SIG_UNBLOCK to remove blocked "ignored signals" may accidentally cause side effects if the calling parent already had said signals blocked to begin with and did not intend to unblock them when creating a pool. Use SIG_SETMASK instead with the previous mask of blocked signals to restore the original blocked set. Co-authored-by: Peter Bierma Co-authored-by: Gregory P. Smith --- Lib/multiprocessing/resource_tracker.py | 7 ++++--- Lib/test/_test_multiprocessing.py | 21 +++++++++++++++++++ ...-12-03-20-28-08.gh-issue-127586.zgotYF.rst | 3 +++ 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 20ddd9c50e3d88..90e036ae905afa 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -155,13 +155,14 @@ def ensure_running(self): # that can make the child die before it registers signal handlers # for SIGINT and SIGTERM. The mask is unregistered after spawning # the child. + prev_sigmask = None try: if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) pid = util.spawnv_passfds(exe, args, fds_to_pass) finally: - if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) + if prev_sigmask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) except: os.close(w) raise diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 80b08b8ac66899..38a03f3391d31d 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6045,6 +6045,27 @@ def test_resource_tracker_exit_code(self): cleanup=cleanup, ) + @unittest.skipUnless(hasattr(signal, "pthread_sigmask"), "pthread_sigmask is not available") + def test_resource_tracker_blocked_signals(self): + # + # gh-127586: Check that resource_tracker does not override blocked signals of caller. + # + from multiprocessing.resource_tracker import ResourceTracker + orig_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, set()) + signals = {signal.SIGTERM, signal.SIGINT, signal.SIGUSR1} + + try: + for sig in signals: + signal.pthread_sigmask(signal.SIG_SETMASK, {sig}) + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker = ResourceTracker() + tracker.ensure_running() + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker._stop() + finally: + # restore sigmask to what it was before executing test + signal.pthread_sigmask(signal.SIG_SETMASK, orig_sigmask) + class TestSimpleQueue(unittest.TestCase): @classmethod diff --git a/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst b/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst new file mode 100644 index 00000000000000..80217bd4a10503 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-03-20-28-08.gh-issue-127586.zgotYF.rst @@ -0,0 +1,3 @@ +:class:`multiprocessing.pool.Pool` now properly restores blocked signal handlers +of the parent thread when creating processes via either *spawn* or +*forkserver*. From aab51c3414bc815c4c31e8ef2a9003f4a546faa9 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 28 Dec 2024 14:59:49 +0000 Subject: [PATCH 302/427] gh-128265: Support WASI/Emscripten on PDB tests, by removing asyncio from pdb tests (#128264) A part of `Lib/test/test_pdb.py` was previously unable to run on WASI/Emscripten platforms because it lacked support for `asyncio`. In fact, these tests could be rewritten without the `asyncio` framework because `test_pdb` tests the behavior of coroutines, which are not part of `asyncio`. Now reliance on the availability of `asyncio` has been removed and part of `test_pdb` that deals with coroutines working on WASI/Emscripten platforms. --- Lib/test/support/__init__.py | 29 +++++++++++ Lib/test/test_contextlib_async.py | 15 +----- Lib/test/test_inspect/test_inspect.py | 15 ++---- Lib/test/test_pdb.py | 71 +++++++++++++-------------- 4 files changed, 70 insertions(+), 60 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 5c738ffaa27713..cf3077f2a4a409 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -62,6 +62,7 @@ "force_not_colorized", "BrokenIter", "in_systemd_nspawn_sync_suppressed", + "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", ] @@ -2940,3 +2941,31 @@ def in_systemd_nspawn_sync_suppressed() -> bool: os.close(fd) return False + +def run_no_yield_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + coro.send(None) + except StopIteration as e: + return e.value + else: + raise AssertionError("coroutine did not complete") + finally: + coro.close() + + +@types.coroutine +def async_yield(v): + return (yield v) + + +def run_yielding_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + while True: + try: + coro.send(None) + except StopIteration as e: + return e.value + finally: + coro.close() diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index d496aa611d1068..7750186e56a5cc 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -3,24 +3,13 @@ asynccontextmanager, AbstractAsyncContextManager, AsyncExitStack, nullcontext, aclosing, contextmanager) from test import support +from test.support import run_no_yield_async_fn as _run_async_fn import unittest import traceback from test.test_contextlib import TestBaseExitStack -def _run_async_fn(async_fn, /, *args, **kwargs): - coro = async_fn(*args, **kwargs) - try: - coro.send(None) - except StopIteration as e: - return e.value - else: - raise AssertionError("coroutine did not complete") - finally: - coro.close() - - def _async_test(async_fn): """Decorator to turn an async function into a synchronous function""" @functools.wraps(async_fn) @@ -546,7 +535,7 @@ def __exit__(self, *exc_details): exit_stack = SyncAsyncExitStack callback_error_internal_frames = [ ('__exit__', 'return _run_async_fn(self.__aexit__, *exc_details)'), - ('_run_async_fn', 'coro.send(None)'), + ('run_no_yield_async_fn', 'coro.send(None)'), ('__aexit__', 'raise exc'), ('__aexit__', 'cb_suppress = cb(*exc_details)'), ] diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index d536d04d2e7d88..345a57a5cfee2d 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -36,6 +36,7 @@ from test.support import cpython_only, import_helper from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ +from test.support import run_no_yield_async_fn from test.support.import_helper import DirsOnSysPath, ready_to_import from test.support.os_helper import TESTFN, temp_cwd from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python @@ -1161,19 +1162,11 @@ def f(self): sys.modules.pop("inspect_actual") def test_nested_class_definition_inside_async_function(self): - def run(coro): - try: - coro.send(None) - except StopIteration as e: - return e.value - else: - raise RuntimeError("coroutine did not complete synchronously!") - finally: - coro.close() + run = run_no_yield_async_fn - self.assertSourceEqual(run(mod2.func225()), 226, 227) + self.assertSourceEqual(run(mod2.func225), 226, 227) self.assertSourceEqual(mod2.cls226, 231, 235) - self.assertSourceEqual(run(mod2.cls226().func232()), 233, 234) + self.assertSourceEqual(run(mod2.cls226().func232), 233, 234) def test_class_definition_same_name_diff_methods(self): self.assertSourceEqual(mod2.cls296, 296, 298) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 9b0806d8b2a9bd..c5ee8c5fb25350 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -20,8 +20,7 @@ from test.support.pty_helper import run_pty, FakeInput from unittest.mock import patch -# gh-114275: WASI fails to run asyncio tests, similar skip than test_asyncio. -SKIP_ASYNCIO_TESTS = (not support.has_socket_support) +SKIP_CORO_TESTS = False class PdbTestInput(object): @@ -1987,7 +1986,7 @@ def test_next_until_return_at_return_event(): """ def test_pdb_next_command_for_generator(): - """Testing skip unwindng stack on yield for generators for "next" command + """Testing skip unwinding stack on yield for generators for "next" command >>> def test_gen(): ... yield 0 @@ -2049,23 +2048,23 @@ def test_pdb_next_command_for_generator(): finished """ -if not SKIP_ASYNCIO_TESTS: +if not SKIP_CORO_TESTS: def test_pdb_next_command_for_coroutine(): - """Testing skip unwindng stack on yield for coroutines for "next" command + """Testing skip unwinding stack on yield for coroutines for "next" command - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def test_coro(): - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) + ... await async_yield(0) + ... await async_yield(0) + ... await async_yield(0) >>> async def test_main(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... await test_coro() >>> def test_function(): - ... asyncio.run(test_main()) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', @@ -2088,13 +2087,13 @@ def test_pdb_next_command_for_coroutine(): -> async def test_coro(): (Pdb) step > (2)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next > (3)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next > (4)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next Internal StopIteration > (3)test_main() @@ -2108,13 +2107,13 @@ def test_pdb_next_command_for_coroutine(): """ def test_pdb_next_command_for_asyncgen(): - """Testing skip unwindng stack on yield for coroutines for "next" command + """Testing skip unwinding stack on yield for coroutines for "next" command - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def agen(): ... yield 1 - ... await asyncio.sleep(0) + ... await async_yield(0) ... yield 2 >>> async def test_coro(): @@ -2126,7 +2125,7 @@ def test_pdb_next_command_for_asyncgen(): ... await test_coro() >>> def test_function(): - ... asyncio.run(test_main()) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', @@ -2163,14 +2162,14 @@ def test_pdb_next_command_for_asyncgen(): -> yield 1 (Pdb) next > (3)agen() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) continue 2 finished """ def test_pdb_return_command_for_generator(): - """Testing no unwindng stack on yield for generators + """Testing no unwinding stack on yield for generators for "return" command >>> def test_gen(): @@ -2228,23 +2227,23 @@ def test_pdb_return_command_for_generator(): finished """ -if not SKIP_ASYNCIO_TESTS: +if not SKIP_CORO_TESTS: def test_pdb_return_command_for_coroutine(): - """Testing no unwindng stack on yield for coroutines for "return" command + """Testing no unwinding stack on yield for coroutines for "return" command - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def test_coro(): - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) + ... await async_yield(0) + ... await async_yield(0) + ... await async_yield(0) >>> async def test_main(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... await test_coro() >>> def test_function(): - ... asyncio.run(test_main()) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', @@ -2264,16 +2263,16 @@ def test_pdb_return_command_for_coroutine(): -> async def test_coro(): (Pdb) step > (2)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next > (3)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) continue finished """ def test_pdb_until_command_for_generator(): - """Testing no unwindng stack on yield for generators + """Testing no unwinding stack on yield for generators for "until" command if target breakpoint is not reached >>> def test_gen(): @@ -2320,20 +2319,20 @@ def test_pdb_until_command_for_generator(): finished """ -if not SKIP_ASYNCIO_TESTS: +if not SKIP_CORO_TESTS: def test_pdb_until_command_for_coroutine(): - """Testing no unwindng stack for coroutines + """Testing no unwinding stack for coroutines for "until" command if target breakpoint is not reached - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def test_coro(): ... print(0) - ... await asyncio.sleep(0) + ... await async_yield(0) ... print(1) - ... await asyncio.sleep(0) + ... await async_yield(0) ... print(2) - ... await asyncio.sleep(0) + ... await async_yield(0) ... print(3) >>> async def test_main(): @@ -2341,7 +2340,7 @@ def test_pdb_until_command_for_coroutine(): ... await test_coro() >>> def test_function(): - ... asyncio.run(test_main()) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', From 2cf396c368a188e9142843e566ce6d8e6eb08999 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Sat, 28 Dec 2024 16:05:00 +0100 Subject: [PATCH 303/427] gh-119786: Fix typos in `InternalDocs/parser.md` (#128314) --- InternalDocs/parser.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index 445b866fc0cb96..be47efe24356d4 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -56,7 +56,7 @@ an input string as its argument, and yields one of the following results: Note that "failure" results do not imply that the program is incorrect, nor do they necessarily mean that the parsing has failed. Since the choice operator is -ordered, a failure very often merely indicates "try the following option". A +ordered, a failure very often merely indicates "try the following option". A direct implementation of a PEG parser as a recursive descent parser will present exponential time performance in the worst case, because PEG parsers have infinite lookahead (this means that they can consider an arbitrary number of @@ -253,7 +253,7 @@ inside curly-braces, which specifies the return value of the alternative: If the action is omitted, a default action is generated: - If there is a single name in the rule, it gets returned. -- If there multiple names in the rule, a collection with all parsed +- If there are multiple names in the rule, a collection with all parsed expressions gets returned (the type of the collection will be different in C and Python). @@ -447,7 +447,7 @@ parser (the one used by the interpreter) just execute: $ make regen-pegen ``` -using the `Makefile` in the main directory. If you are on Windows you can +using the `Makefile` in the main directory. If you are on Windows you can use the Visual Studio project files to regenerate the parser or to execute: ```dos @@ -539,7 +539,7 @@ memoization is used. The C parser used by Python is highly optimized and memoization can be expensive both in memory and time. Although the memory cost is obvious (the parser needs memory for storing previous results in the cache) the execution time cost comes -for continuously checking if the given rule has a cache hit or not. In many +from continuously checking if the given rule has a cache hit or not. In many situations, just parsing it again can be faster. Pegen **disables memoization by default** except for rules with the special marker `memo` after the rule name (and type, if present): @@ -605,7 +605,7 @@ File "", line 1 SyntaxError: invalid syntax ``` -While soft keywords don't have this limitation if used in a context other the +While soft keywords don't have this limitation if used in a context other than one where they are defined as keywords: ```pycon From 492b224b991cd9027f1bc6d9988d01e94f764992 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 28 Dec 2024 21:49:45 +0300 Subject: [PATCH 304/427] gh-128279: Enhance the NetBSD compatibility for thread naming (#128280) Enhance NetBSD compatibility for thread naming in _threadmodule.c. --- Modules/_threadmodule.c | 3 +++ configure | 1 + configure.ac | 1 + 3 files changed, 5 insertions(+) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 75b34a8df7622c..2cbdfeb09b95ae 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -2438,6 +2438,9 @@ _thread_set_name_impl(PyObject *module, PyObject *name_obj) const char *name = PyBytes_AS_STRING(name_encoded); #ifdef __APPLE__ int rc = pthread_setname_np(name); +#elif defined(__NetBSD__) + pthread_t thread = pthread_self(); + int rc = pthread_setname_np(thread, "%s", (void *)name); #else pthread_t thread = pthread_self(); int rc = pthread_setname_np(thread, name); diff --git a/configure b/configure index a697bc1f87b012..299eff6bc3bf70 100755 --- a/configure +++ b/configure @@ -29148,6 +29148,7 @@ CPPFLAGS=$save_CPPFLAGS case "$ac_sys_system" in Linux*) PYTHREAD_NAME_MAXLEN=15;; # Linux and Android SunOS*) PYTHREAD_NAME_MAXLEN=31;; + NetBSD*) PYTHREAD_NAME_MAXLEN=31;; Darwin) PYTHREAD_NAME_MAXLEN=63;; iOS) PYTHREAD_NAME_MAXLEN=63;; FreeBSD*) PYTHREAD_NAME_MAXLEN=98;; diff --git a/configure.ac b/configure.ac index ebc15503f069cc..badb19d55895de 100644 --- a/configure.ac +++ b/configure.ac @@ -7514,6 +7514,7 @@ _RESTORE_VAR([CPPFLAGS]) case "$ac_sys_system" in Linux*) PYTHREAD_NAME_MAXLEN=15;; # Linux and Android SunOS*) PYTHREAD_NAME_MAXLEN=31;; + NetBSD*) PYTHREAD_NAME_MAXLEN=31;; Darwin) PYTHREAD_NAME_MAXLEN=63;; iOS) PYTHREAD_NAME_MAXLEN=63;; FreeBSD*) PYTHREAD_NAME_MAXLEN=98;; From f9a5a3a3ef34e63dc197156e9a5f57842859ca04 Mon Sep 17 00:00:00 2001 From: Calvin Bui <3604363+calvinbui@users.noreply.github.com> Date: Sun, 29 Dec 2024 08:05:34 +1100 Subject: [PATCH 305/427] gh-128192: support HTTP sha-256 digest authentication as per RFC-7617 (GH-128193) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit support sha-256 digest authentication Co-authored-by: Peter Bierma Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Gregory P. Smith --- Doc/library/urllib.request.rst | 3 +++ Doc/whatsnew/3.14.rst | 8 ++++++ Lib/test/test_urllib2.py | 25 ++++++++++++++++--- Lib/urllib/request.py | 7 ++++-- Misc/ACKS | 1 + ...-12-23-11-14-07.gh-issue-128192.02mEhD.rst | 2 ++ 6 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 3c07dc4adf434a..b3efde3f189566 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -411,6 +411,9 @@ The following classes are provided: :ref:`http-password-mgr` for information on the interface that must be supported. + .. versionchanged:: 3.14 + Added support for HTTP digest authentication algorithm ``SHA-256``. + .. class:: HTTPDigestAuthHandler(password_mgr=None) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 935c61c474e889..2767fd3ca48b29 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -646,6 +646,14 @@ unittest (Contributed by Jacob Walls in :gh:`80958`.) +urllib +------ + +* Upgrade HTTP digest authentication algorithm for :mod:`urllib.request` by + supporting SHA-256 digest authentication as specified in :rfc:`7616`. + (Contributed by Calvin Bui in :gh:`128193`.) + + uuid ---- diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 4a9e653515be5b..96d91c1f1c2f8a 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1962,10 +1962,29 @@ def test_parse_proxy(self): self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), - def test_unsupported_algorithm(self): - handler = AbstractDigestAuthHandler() + +class TestDigestAlgorithms(unittest.TestCase): + def setUp(self): + self.handler = AbstractDigestAuthHandler() + + def test_md5_algorithm(self): + H, KD = self.handler.get_algorithm_impls('MD5') + self.assertEqual(H("foo"), "acbd18db4cc2f85cedef654fccc4a4d8") + self.assertEqual(KD("foo", "bar"), "4e99e8c12de7e01535248d2bac85e732") + + def test_sha_algorithm(self): + H, KD = self.handler.get_algorithm_impls('SHA') + self.assertEqual(H("foo"), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + self.assertEqual(KD("foo", "bar"), "54dcbe67d21d5eb39493d46d89ae1f412d3bd6de") + + def test_sha256_algorithm(self): + H, KD = self.handler.get_algorithm_impls('SHA-256') + self.assertEqual(H("foo"), "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + self.assertEqual(KD("foo", "bar"), "a765a8beaa9d561d4c5cbed29d8f4e30870297fdfa9cb7d6e9848a95fec9f937") + + def test_invalid_algorithm(self): with self.assertRaises(ValueError) as exc: - handler.get_algorithm_impls('invalid') + self.handler.get_algorithm_impls('invalid') self.assertEqual( str(exc.exception), "Unsupported digest authentication algorithm 'invalid'" diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index c5a6a18a32bba1..0d1b594b8cf20b 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1048,7 +1048,7 @@ def http_error_407(self, req, fp, code, msg, headers): class AbstractDigestAuthHandler: - # Digest authentication is specified in RFC 2617. + # Digest authentication is specified in RFC 2617/7616. # XXX The client does not inspect the Authentication-Info header # in a successful response. @@ -1176,11 +1176,14 @@ def get_authorization(self, req, chal): return base def get_algorithm_impls(self, algorithm): + # algorithm names taken from RFC 7616 Section 6.1 # lambdas assume digest modules are imported at the top level if algorithm == 'MD5': H = lambda x: hashlib.md5(x.encode("ascii")).hexdigest() - elif algorithm == 'SHA': + elif algorithm == 'SHA': # non-standard, retained for compatibility. H = lambda x: hashlib.sha1(x.encode("ascii")).hexdigest() + elif algorithm == 'SHA-256': + H = lambda x: hashlib.sha256(x.encode("ascii")).hexdigest() # XXX MD5-sess else: raise ValueError("Unsupported digest authentication " diff --git a/Misc/ACKS b/Misc/ACKS index 086930666822ad..c6e53317b37d78 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -258,6 +258,7 @@ Colm Buckley Erik de Bueger Jan-Hein Bührman Marc Bürg +Calvin Bui Lars Buitinck Artem Bulgakov Dick Bulterman diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst new file mode 100644 index 00000000000000..b80ab715ffc7db --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-23-11-14-07.gh-issue-128192.02mEhD.rst @@ -0,0 +1,2 @@ +Upgrade HTTP digest authentication algorithm for :mod:`urllib.request` by +supporting SHA-256 digest authentication as specified in :rfc:`7616`. From c9159b7436363f0cfbddc3deae1bcf925b5c2d4b Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 29 Dec 2024 06:22:29 +0000 Subject: [PATCH 306/427] expand the `asyncio.run_coroutine_threadsafe` recipes (#127576) Co-authored-by: Kumar Aditya --- Doc/library/asyncio-task.rst | 61 +++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f27e858cf420f4..4541cf28de0605 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1067,14 +1067,59 @@ Scheduling From Other Threads This function is meant to be called from a different OS thread than the one where the event loop is running. Example:: - # Create a coroutine - coro = asyncio.sleep(1, result=3) - - # Submit the coroutine to a given loop - future = asyncio.run_coroutine_threadsafe(coro, loop) - - # Wait for the result with an optional timeout argument - assert future.result(timeout) == 3 + def in_thread(loop: asyncio.AbstractEventLoop) -> None: + # Run some blocking IO + pathlib.Path("example.txt").write_text("hello world", encoding="utf8") + + # Create a coroutine + coro = asyncio.sleep(1, result=3) + + # Submit the coroutine to a given loop + future = asyncio.run_coroutine_threadsafe(coro, loop) + + # Wait for the result with an optional timeout argument + assert future.result(timeout=2) == 3 + + async def amain() -> None: + # Get the running loop + loop = asyncio.get_running_loop() + + # Run something in a thread + await asyncio.to_thread(in_thread, loop) + + It's also possible to run the other way around. Example:: + + @contextlib.contextmanager + def loop_in_thread() -> Generator[asyncio.AbstractEventLoop]: + loop_fut = concurrent.futures.Future[asyncio.AbstractEventLoop]() + stop_event = asyncio.Event() + + async def main() -> None: + loop_fut.set_result(asyncio.get_running_loop()) + await stop_event.wait() + + with concurrent.futures.ThreadPoolExecutor(1) as tpe: + complete_fut = tpe.submit(asyncio.run, main()) + for fut in concurrent.futures.as_completed((loop_fut, complete_fut)): + if fut is loop_fut: + loop = loop_fut.result() + try: + yield loop + finally: + loop.call_soon_threadsafe(stop_event.set) + else: + fut.result() + + # Create a loop in another thread + with loop_in_thread() as loop: + # Create a coroutine + coro = asyncio.sleep(1, result=3) + + # Submit the coroutine to a given loop + future = asyncio.run_coroutine_threadsafe(coro, loop) + + # Wait for the result with an optional timeout argument + assert future.result(timeout=2) == 3 If an exception is raised in the coroutine, the returned Future will be notified. It can also be used to cancel the task in From ffece5590e95e89d3b5b6847e3beb443ff65c3db Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 28 Dec 2024 22:32:32 -0800 Subject: [PATCH 307/427] gh-128192: mark new tests with skips based on hashlib algorithm availability (gh-128324) Puts the _hashlib get_fips_mode logic check into test.support rather than spreading it out among other tests. --- Lib/test/support/__init__.py | 8 ++++++++ Lib/test/test_urllib2.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index cf3077f2a4a409..42e7b876594fa7 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2969,3 +2969,11 @@ def run_yielding_async_fn(async_fn, /, *args, **kwargs): return e.value finally: coro.close() + + +def is_libssl_fips_mode(): + try: + from _hashlib import get_fips_mode # ask _hashopenssl.c + except ImportError: + return False # more of a maybe, unless we add this to the _ssl module. + return get_fips_mode() != 0 diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 96d91c1f1c2f8a..085b24c25b2daa 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -27,6 +27,7 @@ import urllib.error import http.client + support.requires_working_socket(module=True) # XXX @@ -1963,20 +1964,29 @@ def test_parse_proxy(self): self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), -class TestDigestAlgorithms(unittest.TestCase): +skip_libssl_fips_mode = unittest.skipIf( + support.is_libssl_fips_mode(), + "conservative skip due to OpenSSL FIPS mode possible algorithm nerfing", +) + + +class TestDigestAuthAlgorithms(unittest.TestCase): def setUp(self): self.handler = AbstractDigestAuthHandler() + @skip_libssl_fips_mode def test_md5_algorithm(self): H, KD = self.handler.get_algorithm_impls('MD5') self.assertEqual(H("foo"), "acbd18db4cc2f85cedef654fccc4a4d8") self.assertEqual(KD("foo", "bar"), "4e99e8c12de7e01535248d2bac85e732") + @skip_libssl_fips_mode def test_sha_algorithm(self): H, KD = self.handler.get_algorithm_impls('SHA') self.assertEqual(H("foo"), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") self.assertEqual(KD("foo", "bar"), "54dcbe67d21d5eb39493d46d89ae1f412d3bd6de") + @skip_libssl_fips_mode def test_sha256_algorithm(self): H, KD = self.handler.get_algorithm_impls('SHA-256') self.assertEqual(H("foo"), "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") From 7e819ce0f32068de7914cd1ba3b4b95e91ea9873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 29 Dec 2024 19:30:53 +0100 Subject: [PATCH 308/427] gh-123424: add `ZipInfo._for_archive` to set suitable default properties (#123429) --------- Co-authored-by: Jason R. Coombs --- Doc/library/zipfile.rst | 11 +++++++ Doc/whatsnew/3.14.rst | 8 +++++ Lib/test/test_zipfile/_path/test_path.py | 19 +---------- Lib/test/test_zipfile/test_core.py | 29 ++++++++++++++++ Lib/zipfile/__init__.py | 33 ++++++++++++------- ...-08-28-16-10-37.gh-issue-123424.u96_i6.rst | 1 + 6 files changed, 72 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 5583c6b24be5c6..afe1cd5c75fcbb 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -84,6 +84,17 @@ The module defines the following items: formerly protected :attr:`!_compresslevel`. The older protected name continues to work as a property for backwards compatibility. + + .. method:: _for_archive(archive) + + Resolve the date_time, compression attributes, and external attributes + to suitable defaults as used by :meth:`ZipFile.writestr`. + + Returns self for chaining. + + .. versionadded:: 3.14 + + .. function:: is_zipfile(filename) Returns ``True`` if *filename* is a valid ZIP file based on its magic number, diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2767fd3ca48b29..53415bb09bf080 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -661,6 +661,14 @@ uuid in :rfc:`9562`. (Contributed by Bénédikt Tran in :gh:`89083`.) +zipinfo +------- + +* Added :func:`ZipInfo._for_archive ` + to resolve suitable defaults for a :class:`~zipfile.ZipInfo` object + as used by :func:`ZipFile.writestr `. + + (Contributed by Bénédikt Tran in :gh:`123424`.) .. Add improved modules above alphabetically, not here at the end. diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index aba515536f0c1a..1ee45f5fc57104 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -634,7 +634,7 @@ def test_backslash_not_separator(self): """ data = io.BytesIO() zf = zipfile.ZipFile(data, "w") - zf.writestr(DirtyZipInfo.for_name("foo\\bar", zf), b"content") + zf.writestr(DirtyZipInfo("foo\\bar")._for_archive(zf), b"content") zf.filename = '' root = zipfile.Path(zf) (first,) = root.iterdir() @@ -657,20 +657,3 @@ class DirtyZipInfo(zipfile.ZipInfo): def __init__(self, filename, *args, **kwargs): super().__init__(filename, *args, **kwargs) self.filename = filename - - @classmethod - def for_name(cls, name, archive): - """ - Construct the same way that ZipFile.writestr does. - - TODO: extract this functionality and re-use - """ - self = cls(filename=name, date_time=time.localtime(time.time())[:6]) - self.compress_type = archive.compression - self.compress_level = archive.compresslevel - if self.filename.endswith('/'): # pragma: no cover - self.external_attr = 0o40775 << 16 # drwxrwxr-x - self.external_attr |= 0x10 # MS-DOS directory flag - else: - self.external_attr = 0o600 << 16 # ?rw------- - return self diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 124e088fd15b80..49f39b9337df85 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -5,6 +5,7 @@ import itertools import os import posixpath +import stat import struct import subprocess import sys @@ -2211,6 +2212,34 @@ def test_create_empty_zipinfo_repr(self): zi = zipfile.ZipInfo(filename="empty") self.assertEqual(repr(zi), "") + def test_for_archive(self): + base_filename = TESTFN2.rstrip('/') + + with zipfile.ZipFile(TESTFN, mode="w", compresslevel=1, + compression=zipfile.ZIP_STORED) as zf: + # no trailing forward slash + zi = zipfile.ZipInfo(base_filename)._for_archive(zf) + self.assertEqual(zi.compress_level, 1) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + # ?rw- --- --- + filemode = stat.S_IRUSR | stat.S_IWUSR + # filemode is stored as the highest 16 bits of external_attr + self.assertEqual(zi.external_attr >> 16, filemode) + self.assertEqual(zi.external_attr & 0xFF, 0) # no MS-DOS flag + + with zipfile.ZipFile(TESTFN, mode="w", compresslevel=1, + compression=zipfile.ZIP_STORED) as zf: + # with a trailing slash + zi = zipfile.ZipInfo(f'{base_filename}/')._for_archive(zf) + self.assertEqual(zi.compress_level, 1) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + # d rwx rwx r-x + filemode = stat.S_IFDIR + filemode |= stat.S_IRWXU | stat.S_IRWXG + filemode |= stat.S_IROTH | stat.S_IXOTH + self.assertEqual(zi.external_attr >> 16, filemode) + self.assertEqual(zi.external_attr & 0xFF, 0x10) # MS-DOS flag + def test_create_empty_zipinfo_default_attributes(self): """Ensure all required attributes are set.""" zi = zipfile.ZipInfo() diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index f4d396abb6e639..052ef47b8f6598 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -13,6 +13,7 @@ import sys import threading import time +from typing import Self try: import zlib # We may need its compression method @@ -605,6 +606,24 @@ def from_file(cls, filename, arcname=None, *, strict_timestamps=True): return zinfo + def _for_archive(self, archive: ZipFile) -> Self: + """Resolve suitable defaults from the archive. + + Resolve the date_time, compression attributes, and external attributes + to suitable defaults as used by :method:`ZipFile.writestr`. + + Return self. + """ + self.date_time = time.localtime(time.time())[:6] + self.compress_type = archive.compression + self.compress_level = archive.compresslevel + if self.filename.endswith('/'): # pragma: no cover + self.external_attr = 0o40775 << 16 # drwxrwxr-x + self.external_attr |= 0x10 # MS-DOS directory flag + else: + self.external_attr = 0o600 << 16 # ?rw------- + return self + def is_dir(self): """Return True if this archive member is a directory.""" if self.filename.endswith('/'): @@ -1908,18 +1927,10 @@ def writestr(self, zinfo_or_arcname, data, the name of the file in the archive.""" if isinstance(data, str): data = data.encode("utf-8") - if not isinstance(zinfo_or_arcname, ZipInfo): - zinfo = ZipInfo(filename=zinfo_or_arcname, - date_time=time.localtime(time.time())[:6]) - zinfo.compress_type = self.compression - zinfo.compress_level = self.compresslevel - if zinfo.filename.endswith('/'): - zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x - zinfo.external_attr |= 0x10 # MS-DOS directory flag - else: - zinfo.external_attr = 0o600 << 16 # ?rw------- - else: + if isinstance(zinfo_or_arcname, ZipInfo): zinfo = zinfo_or_arcname + else: + zinfo = ZipInfo(zinfo_or_arcname)._for_archive(self) if not self.fp: raise ValueError( diff --git a/Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst b/Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst new file mode 100644 index 00000000000000..4df4bbf2ba2b73 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-28-16-10-37.gh-issue-123424.u96_i6.rst @@ -0,0 +1 @@ +Add :meth:`zipfile.ZipInfo._for_archive` setting default properties on :class:`~zipfile.ZipInfo` objects. Patch by Bénédikt Tran and Jason R. Coombs. From c78729f2df7c0e220f1080edb2a0d0cdd49e22df Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 29 Dec 2024 21:42:07 +0000 Subject: [PATCH 309/427] GH-127381: pathlib ABCs: remove `PathBase.stat()` (#128334) Remove the `PathBase.stat()` method. Its use of the `os.stat_result` API, with its 10 mandatory fields and low-level types, makes it an awkward fit for virtual filesystems. We'll look to add a `PathBase.info` attribute later - see GH-125413. --- Lib/pathlib/_abc.py | 31 ++------- Lib/pathlib/_local.py | 12 +++- Lib/test/test_pathlib/test_pathlib.py | 25 ++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 77 +++++++---------------- 4 files changed, 62 insertions(+), 83 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 6acc29ebab2bc5..4d5f7980fd30d2 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -16,7 +16,6 @@ import posixpath from errno import EINVAL from glob import _GlobberBase, _no_recurse_symlinks -from stat import S_ISDIR, S_ISLNK, S_ISREG from pathlib._os import copyfileobj @@ -450,15 +449,6 @@ class PathBase(PurePathBase): """ __slots__ = () - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - raise NotImplementedError - - # Convenience functions for querying the stat results - def exists(self, *, follow_symlinks=True): """ Whether this path exists. @@ -466,39 +456,26 @@ def exists(self, *, follow_symlinks=True): This method normally follows symlinks; to check whether a symlink exists, add the argument follow_symlinks=False. """ - try: - self.stat(follow_symlinks=follow_symlinks) - except (OSError, ValueError): - return False - return True + raise NotImplementedError def is_dir(self, *, follow_symlinks=True): """ Whether this path is a directory. """ - try: - return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) - except (OSError, ValueError): - return False + raise NotImplementedError def is_file(self, *, follow_symlinks=True): """ Whether this path is a regular file (also True for symlinks pointing to regular files). """ - try: - return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) - except (OSError, ValueError): - return False + raise NotImplementedError def is_symlink(self): """ Whether this path is a symbolic link. """ - try: - return S_ISLNK(self.stat(follow_symlinks=False).st_mode) - except (OSError, ValueError): - return False + raise NotImplementedError def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 915402e6c65b29..4484d9553c164f 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -7,7 +7,7 @@ from errno import * from glob import _StringGlobber, _no_recurse_symlinks from itertools import chain -from stat import S_IMODE, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from stat import S_IMODE, S_ISDIR, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from _collections_abc import Sequence try: @@ -725,7 +725,10 @@ def is_dir(self, *, follow_symlinks=True): """ if follow_symlinks: return os.path.isdir(self) - return PathBase.is_dir(self, follow_symlinks=follow_symlinks) + try: + return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False def is_file(self, *, follow_symlinks=True): """ @@ -734,7 +737,10 @@ def is_file(self, *, follow_symlinks=True): """ if follow_symlinks: return os.path.isfile(self) - return PathBase.is_file(self, follow_symlinks=follow_symlinks) + try: + return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False def is_mount(self): """ diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index fac8cbdf65a122..0efa8270210aef 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1835,6 +1835,31 @@ def test_symlink_to_unsupported(self): with self.assertRaises(pathlib.UnsupportedOperation): q.symlink_to(p) + def test_stat(self): + statA = self.cls(self.base).joinpath('fileA').stat() + statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() + statC = self.cls(self.base).joinpath('dirC').stat() + # st_mode: files are the same, directory differs. + self.assertIsInstance(statA.st_mode, int) + self.assertEqual(statA.st_mode, statB.st_mode) + self.assertNotEqual(statA.st_mode, statC.st_mode) + self.assertNotEqual(statB.st_mode, statC.st_mode) + # st_ino: all different, + self.assertIsInstance(statA.st_ino, int) + self.assertNotEqual(statA.st_ino, statB.st_ino) + self.assertNotEqual(statA.st_ino, statC.st_ino) + self.assertNotEqual(statB.st_ino, statC.st_ino) + # st_dev: all the same. + self.assertIsInstance(statA.st_dev, int) + self.assertEqual(statA.st_dev, statB.st_dev) + self.assertEqual(statA.st_dev, statC.st_dev) + # other attributes not used by pathlib. + + def test_stat_no_follow_symlinks_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.stat(follow_symlinks=False)) + @needs_symlinks def test_stat_no_follow_symlinks(self): p = self.cls(self.base) / 'linkA' diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 787fb0f82257e6..81d9d34506f04b 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -2,7 +2,6 @@ import io import os import errno -import stat import unittest from pathlib._abc import PurePathBase, PathBase @@ -1294,11 +1293,6 @@ def close(self): super().close() -DummyPathStatResult = collections.namedtuple( - 'DummyPathStatResult', - 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') - - class DummyPath(PathBase): """ Simple implementation of PathBase that keeps files and directories in @@ -1331,15 +1325,17 @@ def __repr__(self): def with_segments(self, *pathsegments): return type(self)(*pathsegments) - def stat(self, *, follow_symlinks=True): - path = str(self).rstrip('/') - if path in self._files: - st_mode = stat.S_IFREG - elif path in self._directories: - st_mode = stat.S_IFDIR - else: - raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) - return DummyPathStatResult(st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0) + def exists(self, *, follow_symlinks=True): + return self.is_dir() or self.is_file() + + def is_dir(self, *, follow_symlinks=True): + return str(self).rstrip('/') in self._directories + + def is_file(self, *, follow_symlinks=True): + return str(self) in self._files + + def is_symlink(self): + return False def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): @@ -1958,31 +1954,6 @@ def test_rglob_windows(self): self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) - def test_stat(self): - statA = self.cls(self.base).joinpath('fileA').stat() - statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() - statC = self.cls(self.base).joinpath('dirC').stat() - # st_mode: files are the same, directory differs. - self.assertIsInstance(statA.st_mode, int) - self.assertEqual(statA.st_mode, statB.st_mode) - self.assertNotEqual(statA.st_mode, statC.st_mode) - self.assertNotEqual(statB.st_mode, statC.st_mode) - # st_ino: all different, - self.assertIsInstance(statA.st_ino, int) - self.assertNotEqual(statA.st_ino, statB.st_ino) - self.assertNotEqual(statA.st_ino, statC.st_ino) - self.assertNotEqual(statB.st_ino, statC.st_ino) - # st_dev: all the same. - self.assertIsInstance(statA.st_dev, int) - self.assertEqual(statA.st_dev, statB.st_dev) - self.assertEqual(statA.st_dev, statC.st_dev) - # other attributes not used by pathlib. - - def test_stat_no_follow_symlinks_nosymlink(self): - p = self.cls(self.base) / 'fileA' - st = p.stat() - self.assertEqual(st, p.stat(follow_symlinks=False)) - def test_is_dir(self): P = self.cls(self.base) self.assertTrue((P / 'dirA').is_dir()) @@ -2054,26 +2025,26 @@ def test_is_symlink(self): def test_delete_file(self): p = self.cls(self.base) / 'fileA' p._delete() - self.assertFileNotFound(p.stat) + self.assertFalse(p.exists()) self.assertFileNotFound(p._delete) def test_delete_dir(self): base = self.cls(self.base) base.joinpath('dirA')._delete() - self.assertRaises(FileNotFoundError, base.joinpath('dirA').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirA', 'linkC').stat, - follow_symlinks=False) + self.assertFalse(base.joinpath('dirA').exists()) + self.assertFalse(base.joinpath('dirA', 'linkC').exists( + follow_symlinks=False)) base.joinpath('dirB')._delete() - self.assertRaises(FileNotFoundError, base.joinpath('dirB').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'fileB').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'linkD').stat, - follow_symlinks=False) + self.assertFalse(base.joinpath('dirB').exists()) + self.assertFalse(base.joinpath('dirB', 'fileB').exists()) + self.assertFalse(base.joinpath('dirB', 'linkD').exists( + follow_symlinks=False)) base.joinpath('dirC')._delete() - self.assertRaises(FileNotFoundError, base.joinpath('dirC').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD', 'fileD').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'fileC').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'novel.txt').stat) + self.assertFalse(base.joinpath('dirC').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD', 'fileD').exists()) + self.assertFalse(base.joinpath('dirC', 'fileC').exists()) + self.assertFalse(base.joinpath('dirC', 'novel.txt').exists()) def test_delete_missing(self): tmp = self.cls(self.base, 'delete') From ef63cca494571f50906baae1d176469a3dcf8838 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 29 Dec 2024 22:07:12 +0000 Subject: [PATCH 310/427] GH-127381: pathlib ABCs: remove uncommon `PurePathBase` methods (#127853) Remove `PurePathBase.relative_to()` and `is_relative_to()` because they don't account for *other* being an entirely different kind of path, and they can't use `__eq__()` because it's not on the `PurePathBase` interface. Remove `PurePathBase.drive`, `root`, `is_absolute()` and `as_posix()`. These are all too specific to local filesystems. --- Lib/pathlib/_abc.py | 65 ---- Lib/pathlib/_local.py | 5 + Lib/pathlib/_types.py | 2 - Lib/test/test_pathlib/test_pathlib.py | 356 +++++++++++++++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 370 +--------------------- 5 files changed, 365 insertions(+), 433 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 4d5f7980fd30d2..e6ff3fe1187512 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -205,21 +205,6 @@ def __str__(self): passing to system calls.""" raise NotImplementedError - def as_posix(self): - """Return the string representation of the path with forward (/) - slashes.""" - return str(self).replace(self.parser.sep, '/') - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - return self.parser.splitdrive(self.anchor)[0] - - @property - def root(self): - """The root of the path, if any.""" - return self.parser.splitdrive(self.anchor)[1] - @property def anchor(self): """The concatenation of the drive and root, or ''.""" @@ -291,51 +276,6 @@ def with_suffix(self, suffix): else: return self.with_name(stem + suffix) - def relative_to(self, other, *, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if not isinstance(other, PurePathBase): - other = self.with_segments(other) - anchor0, parts0 = _explode_path(self) - anchor1, parts1 = _explode_path(other) - if anchor0 != anchor1: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - while parts0 and parts1 and parts0[-1] == parts1[-1]: - parts0.pop() - parts1.pop() - for part in parts1: - if not part or part == '.': - pass - elif not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - elif part == '..': - raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") - else: - parts0.append('..') - return self.with_segments(*reversed(parts0)) - - def is_relative_to(self, other): - """Return True if the path is relative to another path or False. - """ - if not isinstance(other, PurePathBase): - other = self.with_segments(other) - anchor0, parts0 = _explode_path(self) - anchor1, parts1 = _explode_path(other) - if anchor0 != anchor1: - return False - while parts0 and parts1 and parts0[-1] == parts1[-1]: - parts0.pop() - parts1.pop() - for part in parts1: - if part and part != '.': - return False - return True - @property def parts(self): """An object providing sequence-like access to the @@ -387,11 +327,6 @@ def parents(self): parent = split(path)[0] return tuple(parents) - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - return self.parser.isabs(str(self)) - def match(self, path_pattern, *, case_sensitive=None): """ Return True if this path matches the given pattern. If the pattern is diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 4484d9553c164f..c5721a69d00470 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -437,6 +437,11 @@ def _parse_pattern(cls, pattern): parts.append('') return parts + def as_posix(self): + """Return the string representation of the path with forward (/) + slashes.""" + return str(self).replace(self.parser.sep, '/') + @property def _raw_path(self): paths = self._raw_paths diff --git a/Lib/pathlib/_types.py b/Lib/pathlib/_types.py index baa4a7e2af5b68..72dac2e276fce0 100644 --- a/Lib/pathlib/_types.py +++ b/Lib/pathlib/_types.py @@ -15,7 +15,5 @@ class Parser(Protocol): sep: str def split(self, path: str) -> tuple[str, str]: ... - def splitdrive(self, path: str) -> tuple[str, str]: ... def splitext(self, path: str) -> tuple[str, str]: ... def normcase(self, path: str) -> str: ... - def isabs(self, path: str) -> bool: ... diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 0efa8270210aef..d13daf8ac8cb07 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -270,6 +270,12 @@ def test_as_bytes_common(self): P = self.cls self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') + def test_as_posix_common(self): + P = self.cls + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + self.assertEqual(P(pathstr).as_posix(), pathstr) + # Other tests for as_posix() are in test_equivalences(). + def test_eq_common(self): P = self.cls self.assertEqual(P('a/b'), P('a/b')) @@ -349,6 +355,51 @@ def test_repr_roundtrips(self): self.assertEqual(q, p) self.assertEqual(repr(q), r) + def test_drive_common(self): + P = self.cls + self.assertEqual(P('a/b').drive, '') + self.assertEqual(P('/a/b').drive, '') + self.assertEqual(P('').drive, '') + + @needs_windows + def test_drive_windows(self): + P = self.cls + self.assertEqual(P('c:').drive, 'c:') + self.assertEqual(P('c:a/b').drive, 'c:') + self.assertEqual(P('c:/').drive, 'c:') + self.assertEqual(P('c:/a/b/').drive, 'c:') + self.assertEqual(P('//a/b').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') + + + def test_root_common(self): + P = self.cls + sep = self.sep + self.assertEqual(P('').root, '') + self.assertEqual(P('a/b').root, '') + self.assertEqual(P('/').root, sep) + self.assertEqual(P('/a/b').root, sep) + + @needs_posix + def test_root_posix(self): + P = self.cls + self.assertEqual(P('/a/b').root, '/') + # POSIX special case for two leading slashes. + self.assertEqual(P('//a/b').root, '//') + + @needs_windows + def test_root_windows(self): + P = self.cls + self.assertEqual(P('c:').root, '') + self.assertEqual(P('c:a/b').root, '') + self.assertEqual(P('c:/').root, '\\') + self.assertEqual(P('c:/a/b/').root, '\\') + self.assertEqual(P('//a/b').root, '\\') + self.assertEqual(P('//a/b/').root, '\\') + self.assertEqual(P('//a/b/c/d').root, '\\') + def test_name_empty(self): P = self.cls self.assertEqual(P('').name, '') @@ -547,6 +598,311 @@ def assertOrderedEqual(a, b): self.assertFalse(p < q) self.assertFalse(p > q) + @needs_posix + def test_is_absolute_posix(self): + P = self.cls + self.assertFalse(P('').is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertTrue(P('/').is_absolute()) + self.assertTrue(P('/a').is_absolute()) + self.assertTrue(P('/a/b/').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + + @needs_windows + def test_is_absolute_windows(self): + P = self.cls + # Under NT, only paths with both a drive and a root are absolute. + self.assertFalse(P().is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertFalse(P('/').is_absolute()) + self.assertFalse(P('/a').is_absolute()) + self.assertFalse(P('/a/b/').is_absolute()) + self.assertFalse(P('c:').is_absolute()) + self.assertFalse(P('c:a').is_absolute()) + self.assertFalse(P('c:a/b/').is_absolute()) + self.assertTrue(P('c:/').is_absolute()) + self.assertTrue(P('c:/a').is_absolute()) + self.assertTrue(P('c:/a/b/').is_absolute()) + # UNC paths are absolute by definition. + self.assertTrue(P('//').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + self.assertTrue(P('//a/b/').is_absolute()) + self.assertTrue(P('//a/b/c').is_absolute()) + self.assertTrue(P('//a/b/c/d').is_absolute()) + self.assertTrue(P('//?/UNC/').is_absolute()) + self.assertTrue(P('//?/UNC/spam').is_absolute()) + + def test_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.relative_to) + self.assertRaises(TypeError, p.relative_to, b'a') + self.assertEqual(p.relative_to(P('')), P('a/b')) + self.assertEqual(p.relative_to(''), P('a/b')) + self.assertEqual(p.relative_to(P('a')), P('b')) + self.assertEqual(p.relative_to('a'), P('b')) + self.assertEqual(p.relative_to('a/'), P('b')) + self.assertEqual(p.relative_to(P('a/b')), P('')) + self.assertEqual(p.relative_to('a/b'), P('')) + self.assertEqual(p.relative_to(P(''), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('c')) + self.assertRaises(ValueError, p.relative_to, P('a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('a/c')) + self.assertRaises(ValueError, p.relative_to, P('/a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + p = P('/a/b') + self.assertEqual(p.relative_to(P('/')), P('a/b')) + self.assertEqual(p.relative_to('/'), P('a/b')) + self.assertEqual(p.relative_to(P('/a')), P('b')) + self.assertEqual(p.relative_to('/a'), P('b')) + self.assertEqual(p.relative_to('/a/'), P('b')) + self.assertEqual(p.relative_to(P('/a/b')), P('')) + self.assertEqual(p.relative_to('/a/b'), P('')) + self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('/a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/c')) + self.assertRaises(ValueError, p.relative_to, P('')) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + + @needs_windows + def test_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) + self.assertEqual(p.relative_to('c:foO'), P('Bar')) + self.assertEqual(p.relative_to('c:foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR')), P()) + self.assertEqual(p.relative_to('c:foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) + self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P()) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) + self.assertRaises(ValueError, p.relative_to, '', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) + p = P('C:/Foo/Bar') + self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) + self.assertEqual(p.relative_to('c:/foO'), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) + self.assertEqual(p.relative_to('c:/foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) + self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, 'c:') + self.assertRaises(ValueError, p.relative_to, P('c:')) + self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo')) + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('d:/')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) + self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) + + def test_is_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.is_relative_to) + self.assertRaises(TypeError, p.is_relative_to, b'a') + self.assertTrue(p.is_relative_to(P(''))) + self.assertTrue(p.is_relative_to('')) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to('a/')) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertTrue(p.is_relative_to('a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + p = P('/a/b') + self.assertTrue(p.is_relative_to(P('/'))) + self.assertTrue(p.is_relative_to('/')) + self.assertTrue(p.is_relative_to(P('/a'))) + self.assertTrue(p.is_relative_to('/a')) + self.assertTrue(p.is_relative_to('/a/')) + self.assertTrue(p.is_relative_to(P('/a/b'))) + self.assertTrue(p.is_relative_to('/a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/c'))) + self.assertFalse(p.is_relative_to(P('/a/b/c'))) + self.assertFalse(p.is_relative_to(P('/a/c'))) + self.assertFalse(p.is_relative_to(P(''))) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('a'))) + + @needs_windows + def test_is_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:'))) + self.assertTrue(p.is_relative_to('c:')) + self.assertTrue(p.is_relative_to(P('c:foO'))) + self.assertTrue(p.is_relative_to('c:foO')) + self.assertTrue(p.is_relative_to('c:foO/')) + self.assertTrue(p.is_relative_to(P('c:foO/baR'))) + self.assertTrue(p.is_relative_to('c:foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('Foo'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('C:/Foo'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) + p = P('C:/Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:/'))) + self.assertTrue(p.is_relative_to(P('c:/foO'))) + self.assertTrue(p.is_relative_to('c:/foO/')) + self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) + self.assertTrue(p.is_relative_to('c:/foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to('c:')) + self.assertFalse(p.is_relative_to(P('C:/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo'))) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('d:/'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('//C/Foo'))) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) + self.assertTrue(p.is_relative_to('//sErver/sHare')) + self.assertTrue(p.is_relative_to('//sErver/sHare/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) + class PurePosixPathTest(PurePathTest): cls = pathlib.PurePosixPath diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 81d9d34506f04b..d588442bd11785 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -71,7 +71,7 @@ def __hash__(self): return hash(str(self)) def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + return "{}({!r})".format(self.__class__.__name__, str(self)) def with_segments(self, *pathsegments): return type(self)(*pathsegments) @@ -107,12 +107,6 @@ def test_constructor_common(self): P('a/b/c') P('/a/b/c') - def test_fspath_common(self): - self.assertRaises(TypeError, os.fspath, self.cls('')) - - def test_as_bytes_common(self): - self.assertRaises(TypeError, bytes, self.cls('')) - def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object # from a str subclass instance, and it then gets converted to @@ -161,7 +155,6 @@ def with_segments(self, *pathsegments): self.assertEqual(42, p.with_stem('foo').session_id) self.assertEqual(42, p.with_suffix('.foo').session_id) self.assertEqual(42, p.with_segments('foo').session_id) - self.assertEqual(42, p.relative_to('foo').session_id) self.assertEqual(42, p.parent.session_id) for parent in p.parents: self.assertEqual(42, parent.session_id) @@ -303,12 +296,6 @@ def test_str_windows(self): p = self.cls('//a/b/c/d') self.assertEqual(str(p), '\\\\a\\b\\c\\d') - def test_as_posix_common(self): - P = self.cls - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - self.assertEqual(P(pathstr).as_posix(), pathstr) - # Other tests for as_posix() are in test_equivalences(). - def test_match_empty(self): P = self.cls self.assertRaises(ValueError, P('a').match, '') @@ -615,50 +602,6 @@ def test_parents_windows(self): with self.assertRaises(IndexError): par[2] - def test_drive_common(self): - P = self.cls - self.assertEqual(P('a/b').drive, '') - self.assertEqual(P('/a/b').drive, '') - self.assertEqual(P('').drive, '') - - @needs_windows - def test_drive_windows(self): - P = self.cls - self.assertEqual(P('c:').drive, 'c:') - self.assertEqual(P('c:a/b').drive, 'c:') - self.assertEqual(P('c:/').drive, 'c:') - self.assertEqual(P('c:/a/b/').drive, 'c:') - self.assertEqual(P('//a/b').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') - self.assertEqual(P('./c:a').drive, '') - - def test_root_common(self): - P = self.cls - sep = self.sep - self.assertEqual(P('').root, '') - self.assertEqual(P('a/b').root, '') - self.assertEqual(P('/').root, sep) - self.assertEqual(P('/a/b').root, sep) - - @needs_posix - def test_root_posix(self): - P = self.cls - self.assertEqual(P('/a/b').root, '/') - # POSIX special case for two leading slashes. - self.assertEqual(P('//a/b').root, '//') - - @needs_windows - def test_root_windows(self): - P = self.cls - self.assertEqual(P('c:').root, '') - self.assertEqual(P('c:a/b').root, '') - self.assertEqual(P('c:/').root, '\\') - self.assertEqual(P('c:/a/b/').root, '\\') - self.assertEqual(P('//a/b').root, '\\') - self.assertEqual(P('//a/b/').root, '\\') - self.assertEqual(P('//a/b/c/d').root, '\\') - def test_anchor_common(self): P = self.cls sep = self.sep @@ -967,311 +910,6 @@ def test_with_suffix_invalid(self): self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') self.assertRaises(TypeError, P('a/b').with_suffix, None) - def test_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.relative_to) - self.assertRaises(TypeError, p.relative_to, b'a') - self.assertEqual(p.relative_to(P('')), P('a/b')) - self.assertEqual(p.relative_to(''), P('a/b')) - self.assertEqual(p.relative_to(P('a')), P('b')) - self.assertEqual(p.relative_to('a'), P('b')) - self.assertEqual(p.relative_to('a/'), P('b')) - self.assertEqual(p.relative_to(P('a/b')), P('')) - self.assertEqual(p.relative_to('a/b'), P('')) - self.assertEqual(p.relative_to(P(''), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) - self.assertEqual(p.relative_to('a/b', walk_up=True), P('')) - self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('c')) - self.assertRaises(ValueError, p.relative_to, P('a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('a/c')) - self.assertRaises(ValueError, p.relative_to, P('/a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - p = P('/a/b') - self.assertEqual(p.relative_to(P('/')), P('a/b')) - self.assertEqual(p.relative_to('/'), P('a/b')) - self.assertEqual(p.relative_to(P('/a')), P('b')) - self.assertEqual(p.relative_to('/a'), P('b')) - self.assertEqual(p.relative_to('/a/'), P('b')) - self.assertEqual(p.relative_to(P('/a/b')), P('')) - self.assertEqual(p.relative_to('/a/b'), P('')) - self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P('')) - self.assertEqual(p.relative_to('/a/b', walk_up=True), P('')) - self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/c')) - self.assertRaises(ValueError, p.relative_to, P('')) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - - @needs_windows - def test_relative_to_windows(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) - self.assertEqual(p.relative_to('c:foO'), P('Bar')) - self.assertEqual(p.relative_to('c:foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR')), P()) - self.assertEqual(p.relative_to('c:foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) - self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) - self.assertRaises(ValueError, p.relative_to, '', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) - p = P('C:/Foo/Bar') - self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) - self.assertEqual(p.relative_to('c:/foO'), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) - self.assertEqual(p.relative_to('c:/foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) - self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, 'c:') - self.assertRaises(ValueError, p.relative_to, P('c:')) - self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo')) - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('d:/')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) - self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) - - def test_is_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.is_relative_to) - self.assertRaises(TypeError, p.is_relative_to, b'a') - self.assertTrue(p.is_relative_to(P(''))) - self.assertTrue(p.is_relative_to('')) - self.assertTrue(p.is_relative_to(P('a'))) - self.assertTrue(p.is_relative_to('a/')) - self.assertTrue(p.is_relative_to(P('a/b'))) - self.assertTrue(p.is_relative_to('a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('c'))) - self.assertFalse(p.is_relative_to(P('a/b/c'))) - self.assertFalse(p.is_relative_to(P('a/c'))) - self.assertFalse(p.is_relative_to(P('/a'))) - p = P('/a/b') - self.assertTrue(p.is_relative_to(P('/'))) - self.assertTrue(p.is_relative_to('/')) - self.assertTrue(p.is_relative_to(P('/a'))) - self.assertTrue(p.is_relative_to('/a')) - self.assertTrue(p.is_relative_to('/a/')) - self.assertTrue(p.is_relative_to(P('/a/b'))) - self.assertTrue(p.is_relative_to('/a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/c'))) - self.assertFalse(p.is_relative_to(P('/a/b/c'))) - self.assertFalse(p.is_relative_to(P('/a/c'))) - self.assertFalse(p.is_relative_to(P(''))) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('a'))) - - @needs_windows - def test_is_relative_to_windows(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:'))) - self.assertTrue(p.is_relative_to('c:')) - self.assertTrue(p.is_relative_to(P('c:foO'))) - self.assertTrue(p.is_relative_to('c:foO')) - self.assertTrue(p.is_relative_to('c:foO/')) - self.assertTrue(p.is_relative_to(P('c:foO/baR'))) - self.assertTrue(p.is_relative_to('c:foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('Foo'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('C:/Foo'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) - p = P('C:/Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:/'))) - self.assertTrue(p.is_relative_to(P('c:/foO'))) - self.assertTrue(p.is_relative_to('c:/foO/')) - self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) - self.assertTrue(p.is_relative_to('c:/foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to('c:')) - self.assertFalse(p.is_relative_to(P('C:/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo'))) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('d:/'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('//C/Foo'))) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) - self.assertTrue(p.is_relative_to('//sErver/sHare')) - self.assertTrue(p.is_relative_to('//sErver/sHare/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) - - @needs_posix - def test_is_absolute_posix(self): - P = self.cls - self.assertFalse(P('').is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertTrue(P('/').is_absolute()) - self.assertTrue(P('/a').is_absolute()) - self.assertTrue(P('/a/b/').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - - @needs_windows - def test_is_absolute_windows(self): - P = self.cls - # Under NT, only paths with both a drive and a root are absolute. - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertFalse(P('/').is_absolute()) - self.assertFalse(P('/a').is_absolute()) - self.assertFalse(P('/a/b/').is_absolute()) - self.assertFalse(P('c:').is_absolute()) - self.assertFalse(P('c:a').is_absolute()) - self.assertFalse(P('c:a/b/').is_absolute()) - self.assertTrue(P('c:/').is_absolute()) - self.assertTrue(P('c:/a').is_absolute()) - self.assertTrue(P('c:/a/b/').is_absolute()) - # UNC paths are absolute by definition. - self.assertTrue(P('//').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - self.assertTrue(P('//a/b/').is_absolute()) - self.assertTrue(P('//a/b/c').is_absolute()) - self.assertTrue(P('//a/b/c/d').is_absolute()) - self.assertTrue(P('//?/UNC/').is_absolute()) - self.assertTrue(P('//?/UNC/spam').is_absolute()) - # # Tests for the virtual classes. @@ -1320,7 +958,7 @@ def __hash__(self): return hash(str(self)) def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + return "{}({!r})".format(self.__class__.__name__, str(self)) def with_segments(self, *pathsegments): return type(self)(*pathsegments) @@ -1631,8 +1269,8 @@ def ordered_walk(path): source_walk = ordered_walk(source) target_walk = ordered_walk(target) for source_item, target_item in zip(source_walk, target_walk, strict=True): - self.assertEqual(source_item[0].relative_to(source), - target_item[0].relative_to(target)) # dirpath + self.assertEqual(source_item[0].parts[len(source.parts):], + target_item[0].parts[len(target.parts):]) # dirpath self.assertEqual(source_item[1], target_item[1]) # dirnames self.assertEqual(source_item[2], target_item[2]) # filenames # Compare files and symlinks From 81376fef76a53fb79893bfa9c9db18d97c228fbe Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Mon, 30 Dec 2024 02:55:14 -0600 Subject: [PATCH 311/427] gh-128321: Set LIBS instead of LDFLAGS when checking sqlite3 requirements (#128322) Co-authored-by: Erlend E. Aasland --- .../next/Build/2024-12-28-21-05-19.gh-issue-128321.0UvbXw.rst | 3 +++ configure | 2 +- configure.ac | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-12-28-21-05-19.gh-issue-128321.0UvbXw.rst diff --git a/Misc/NEWS.d/next/Build/2024-12-28-21-05-19.gh-issue-128321.0UvbXw.rst b/Misc/NEWS.d/next/Build/2024-12-28-21-05-19.gh-issue-128321.0UvbXw.rst new file mode 100644 index 00000000000000..ed72cc8ab1449a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-28-21-05-19.gh-issue-128321.0UvbXw.rst @@ -0,0 +1,3 @@ +Set ``LIBS`` instead of ``LDFLAGS`` when checking if :mod:`sqlite3` library +functions are available. This fixes the ordering of linked libraries during +checks, which was incorrect when using a statically linked ``libsqlite3``. diff --git a/configure b/configure index 299eff6bc3bf70..3d2c60213db591 100755 --- a/configure +++ b/configure @@ -15434,7 +15434,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBSQLITE3_CFLAGS" - LDFLAGS="$LIBSQLITE3_LIBS $LDFLAGS" + LIBS="$LIBSQLITE3_LIBS $LIBS" ac_fn_c_check_header_compile "$LINENO" "sqlite3.h" "ac_cv_header_sqlite3_h" "$ac_includes_default" if test "x$ac_cv_header_sqlite3_h" = xyes diff --git a/configure.ac b/configure.ac index badb19d55895de..ee034e5a9621df 100644 --- a/configure.ac +++ b/configure.ac @@ -4220,7 +4220,7 @@ WITH_SAVE_ENV([ dnl bpo-45774/GH-29507: The CPP check in AC_CHECK_HEADER can fail on FreeBSD, dnl hence CPPFLAGS instead of CFLAGS. CPPFLAGS="$CPPFLAGS $LIBSQLITE3_CFLAGS" - LDFLAGS="$LIBSQLITE3_LIBS $LDFLAGS" + LIBS="$LIBSQLITE3_LIBS $LIBS" AC_CHECK_HEADER([sqlite3.h], [ have_sqlite3=yes From 2bd5a7ab0f4a1f65ab8043001bd6e8416c5079bd Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 30 Dec 2024 13:53:40 +0100 Subject: [PATCH 312/427] Docs: correctly markup sys.monitoring "What's New" entry (#128346) The sys.monitoring entry was added with commit d2f1d917e. --- Doc/whatsnew/3.14.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 53415bb09bf080..63fa21e17bc834 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -614,8 +614,8 @@ sys sys.monitoring -------------- -Two new events are added: :monitoring-event:`BRANCH_LEFT` and -:monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. +* Two new events are added: :monitoring-event:`BRANCH_LEFT` and + :monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. tkinter ------- From 348012432155271cfbee71a78d0c27254fc230e2 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 30 Dec 2024 08:19:38 -0800 Subject: [PATCH 313/427] gh-119180: Set the name of the param to __annotate__ to "format" (#124730) --- Lib/test/test_pydoc/test_pydoc.py | 4 +-- Lib/test/test_type_annotations.py | 49 +++++++++++++++++++++++++++++++ Python/codegen.c | 27 +++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 3283fde9e12a8a..c798b11f5aa56e 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -79,7 +79,7 @@ class A(builtins.object) class B(builtins.object) | Methods defined here: | - | __annotate__(...) + | __annotate__(format, /) | | ---------------------------------------------------------------------- | Data descriptors defined here: @@ -180,7 +180,7 @@ class A(builtins.object) class B(builtins.object) Methods defined here: - __annotate__(...) + __annotate__(format, /) ---------------------------------------------------------------------- Data descriptors defined here: __dict__ diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 7d88f4cdfa3141..0afcd76af153e7 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,4 +1,5 @@ import annotationlib +import inspect import textwrap import types import unittest @@ -380,6 +381,11 @@ class X: annotate(None) self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) + sig = inspect.signature(annotate) + self.assertEqual(sig, inspect.Signature([ + inspect.Parameter("format", inspect.Parameter.POSITIONAL_ONLY) + ])) + def test_comprehension_in_annotation(self): # This crashed in an earlier version of the code ns = run_code("x: [y for y in range(10)]") @@ -400,6 +406,7 @@ def f(x: int) -> int: pass def test_name_clash_with_format(self): # this test would fail if __annotate__'s parameter was called "format" + # during symbol table construction code = """ class format: pass @@ -408,3 +415,45 @@ def f(x: format): pass ns = run_code(code) f = ns["f"] self.assertEqual(f.__annotations__, {"x": ns["format"]}) + + code = """ + class Outer: + class format: pass + + def meth(self, x: format): ... + """ + ns = run_code(code) + self.assertEqual(ns["Outer"].meth.__annotations__, {"x": ns["Outer"].format}) + + code = """ + def f(format): + def inner(x: format): pass + return inner + res = f("closure var") + """ + ns = run_code(code) + self.assertEqual(ns["res"].__annotations__, {"x": "closure var"}) + + code = """ + def f(x: format): + pass + """ + ns = run_code(code) + # picks up the format() builtin + self.assertEqual(ns["f"].__annotations__, {"x": format}) + + code = """ + def outer(): + def f(x: format): + pass + if False: + class format: pass + return f + f = outer() + """ + ns = run_code(code) + with self.assertRaisesRegex( + NameError, + "cannot access free variable 'format' where it is not associated with a value in enclosing scope", + ): + ns["f"].__annotations__ diff --git a/Python/codegen.c b/Python/codegen.c index 6d3272edfdbf94..7432415b17414e 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -701,6 +701,33 @@ codegen_leave_annotations_scope(compiler *c, location loc, ADDOP_I(c, loc, BUILD_MAP, annotations_len); ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1); + + // We want the parameter to __annotate__ to be named "format" in the + // signature shown by inspect.signature(), but we need to use a + // different name (.format) in the symtable; if the name + // "format" appears in the annotations, it doesn't get clobbered + // by this name. This code is essentially: + // co->co_localsplusnames = ("format", *co->co_localsplusnames[1:]) + const Py_ssize_t size = PyObject_Size(co->co_localsplusnames); + if (size == -1) { + return ERROR; + } + PyObject *new_names = PyTuple_New(size); + if (new_names == NULL) { + return ERROR; + } + PyTuple_SET_ITEM(new_names, 0, Py_NewRef(&_Py_ID(format))); + for (int i = 1; i < size; i++) { + PyObject *item = PyTuple_GetItem(co->co_localsplusnames, i); + if (item == NULL) { + Py_DECREF(new_names); + return ERROR; + } + Py_INCREF(item); + PyTuple_SET_ITEM(new_names, i, item); + } + Py_SETREF(co->co_localsplusnames, new_names); + _PyCompile_ExitScope(c); if (co == NULL) { return ERROR; From 34b85ef26c44bbbd8a05d3e7dee6997b3623aab0 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 30 Dec 2024 18:18:42 +0100 Subject: [PATCH 314/427] gh-128118: Speed up copy.copy with fast lookup for atomic and container types (#128119) --- Lib/copy.py | 27 +++++++------------ ...-12-20-10-57-10.gh-issue-128118.mYak8i.rst | 2 ++ 2 files changed, 11 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-20-10-57-10.gh-issue-128118.mYak8i.rst diff --git a/Lib/copy.py b/Lib/copy.py index f27e109973cfb7..c64fc0761793f5 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -67,13 +67,15 @@ def copy(x): cls = type(x) - copier = _copy_dispatch.get(cls) - if copier: - return copier(x) + if cls in _copy_atomic_types: + return x + if cls in _copy_builtin_containers: + return cls.copy(x) + if issubclass(cls, type): # treat it as a regular class: - return _copy_immutable(x) + return x copier = getattr(cls, "__copy__", None) if copier is not None: @@ -98,23 +100,12 @@ def copy(x): return _reconstruct(x, None, *rv) -_copy_dispatch = d = {} - -def _copy_immutable(x): - return x -for t in (types.NoneType, int, float, bool, complex, str, tuple, +_copy_atomic_types = {types.NoneType, int, float, bool, complex, str, tuple, bytes, frozenset, type, range, slice, property, types.BuiltinFunctionType, types.EllipsisType, types.NotImplementedType, types.FunctionType, types.CodeType, - weakref.ref, super): - d[t] = _copy_immutable - -d[list] = list.copy -d[dict] = dict.copy -d[set] = set.copy -d[bytearray] = bytearray.copy - -del d, t + weakref.ref, super} +_copy_builtin_containers = {list, dict, set, bytearray} def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. diff --git a/Misc/NEWS.d/next/Library/2024-12-20-10-57-10.gh-issue-128118.mYak8i.rst b/Misc/NEWS.d/next/Library/2024-12-20-10-57-10.gh-issue-128118.mYak8i.rst new file mode 100644 index 00000000000000..bc2898edfda721 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-20-10-57-10.gh-issue-128118.mYak8i.rst @@ -0,0 +1,2 @@ +Improve performance of :func:`copy.copy` by 30% via +a fast path for atomic types and container types. From 6dbace397fab5154f66226abb11af32d2d13219f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 30 Dec 2024 19:25:39 +0200 Subject: [PATCH 315/427] gh-128317: Document `calendar.TextCalendar.formatweek` (#128353) --- Doc/library/calendar.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index eafc038d6cb722..9a4395ecb5084d 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -138,6 +138,14 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :class:`TextCalendar` instances have the following methods: + .. method:: formatweek(theweek, w=0) + + Return a single week in a string with no newline. If *w* is provided, it + specifies the width of the date columns, which are centered. Depends + on the first weekday as specified in the constructor or set by the + :meth:`setfirstweekday` method. + + .. method:: formatmonth(theyear, themonth, w=0, l=0) Return a month's calendar in a multi-line string. If *w* is provided, it From fe4dd07a84ba423179a93ed84bdcc2b4c99b35a9 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Mon, 30 Dec 2024 19:38:09 +0100 Subject: [PATCH 316/427] gh-119786: Mention `InternalDocs/interpreter.md` instead of non-existing `adaptive.md` (#128329) `Python/specialize.c`: Mention `InternalDocs/interpreter.md` instead of non-existing `adaptive.md` Co-authored-by: Peter Bierma --- Python/specialize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/specialize.c b/Python/specialize.c index 349ed472298945..2148b62c3a1953 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -21,7 +21,7 @@ extern const char *_PyUOpName(int index); /* For guidance on adding or extending families of instructions see - * ./adaptive.md + * InternalDocs/interpreter.md `Specialization` section. */ #ifdef Py_STATS From 47d2cb8eb7595df5940225867dbb66b6dd59413a Mon Sep 17 00:00:00 2001 From: Bogdan Romanyuk <65823030+wrongnull@users.noreply.github.com> Date: Mon, 30 Dec 2024 23:38:49 +0300 Subject: [PATCH 317/427] gh-128100: Use atomic dictionary load in `_PyObject_GenericGetAttrWithDict` (GH-128297) --- Objects/object.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Objects/object.c b/Objects/object.c index d584414c559b9d..4c30257ca26938 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1717,7 +1717,11 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); if (dictptr) { +#ifdef Py_GIL_DISABLED + dict = _Py_atomic_load_ptr_acquire(dictptr); +#else dict = *dictptr; +#endif } } } From dafe7a44630aa32bb411cceb45c7b7df725e3fe3 Mon Sep 17 00:00:00 2001 From: n-l-i <57808975+n-l-i@users.noreply.github.com> Date: Mon, 30 Dec 2024 21:52:04 +0100 Subject: [PATCH 318/427] gh-128342: Specify timeout unit in subprocess docstrings (GH-128343) Specify timeout unit (seconds) in subprocess docstrings Co-authored-by: Terry Jan Reedy --- Lib/subprocess.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 88f0230b05fbc7..de88eedcf80ff9 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -386,7 +386,7 @@ def _text_encoding(): def call(*popenargs, timeout=None, **kwargs): """Run command with arguments. Wait for command to complete or - timeout, then return the returncode attribute. + for timeout seconds, then return the returncode attribute. The arguments are the same as for the Popen constructor. Example: @@ -523,8 +523,8 @@ def run(*popenargs, in the returncode attribute, and output & stderr attributes if those streams were captured. - If timeout is given, and the process takes too long, a TimeoutExpired - exception will be raised. + If timeout (seconds) is given and the process takes too long, + a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument From 7ef49074123511003c8b7f7f3ba2a4e05285e8dc Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Tue, 31 Dec 2024 12:24:17 +0800 Subject: [PATCH 319/427] gh-128262: Allow specialization of calls to classes with __slots__ (GH-128263) --- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 6 ++++-- Python/executor_cases.c.h | 8 ++++++-- Python/generated_cases.c.h | 8 ++++++-- Python/specialize.c | 4 ---- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index e71194b116e020..73fc29eb78a7a4 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -234,7 +234,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 63cf1978e8abe5..602cf7f47b812b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3765,13 +3765,15 @@ dummy_func( DEOPT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version); - assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)); STAT_INC(CALL, hit); - PyObject *self_o = _PyType_NewManagedObject(tp); + PyObject *self_o = PyType_GenericAlloc(tp, 0); if (self_o == NULL) { ERROR_NO_POP(); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 22335021faaa6d..f7374d52705960 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4572,7 +4572,9 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; @@ -4581,7 +4583,9 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - PyObject *self_o = _PyType_NewManagedObject(tp); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); if (self_o == NULL) { JUMP_TO_ERROR(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index bed16b60b76a2f..98743c27c38524 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1048,13 +1048,17 @@ DEOPT_IF(!PyType_Check(callable_o), CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); - assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); STAT_INC(CALL, hit); - PyObject *self_o = _PyType_NewManagedObject(tp); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); if (self_o == NULL) { goto error; } diff --git a/Python/specialize.c b/Python/specialize.c index 2148b62c3a1953..c918c77779d20d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2002,10 +2002,6 @@ get_init_for_simple_managed_python_class(PyTypeObject *tp, unsigned int *tp_vers return NULL; } unsigned long tp_flags = PyType_GetFlags(tp); - if ((tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES); - return NULL; - } if (!(tp_flags & Py_TPFLAGS_HEAPTYPE)) { /* Is this possible? */ SPECIALIZATION_FAIL(CALL, SPEC_FAIL_EXPECTED_ERROR); From 7c72c1f0dfff8950e3c6f656307099f1b0a4f8a3 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 31 Dec 2024 11:50:35 +0530 Subject: [PATCH 320/427] gh-128277: use relaxed atomics for `sock_fd` (#128304) --- Modules/socketmodule.c | 167 +++++++++++++++++++++++++---------------- 1 file changed, 102 insertions(+), 65 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 1e95be9b1bc9f4..1f0cbf6abb184c 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -110,6 +110,7 @@ Local naming conventions: #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState #include "pycore_time.h" // _PyTime_AsMilliseconds() +#include "pycore_pyatomic_ft_wrappers.h" #ifdef _Py_MEMORY_SANITIZER # include @@ -564,6 +565,42 @@ typedef struct _socket_state { #endif } socket_state; +static inline void +set_sock_fd(PySocketSockObject *s, SOCKET_T fd) +{ +#ifdef Py_GIL_DISABLED +#if SIZEOF_SOCKET_T == SIZEOF_INT + _Py_atomic_store_int_relaxed((int *)&s->sock_fd, (int)fd); +#elif SIZEOF_SOCKET_T == SIZEOF_LONG + _Py_atomic_store_long_relaxed((long *)&s->sock_fd, (long)fd); +#elif SIZEOF_SOCKET_T == SIZEOF_LONG_LONG + _Py_atomic_store_llong_relaxed((long long *)&s->sock_fd, (long long)fd); +#else + #error "Unsupported SIZEOF_SOCKET_T" +#endif +#else + s->sock_fd = fd; +#endif +} + +static inline SOCKET_T +get_sock_fd(PySocketSockObject *s) +{ +#ifdef Py_GIL_DISABLED +#if SIZEOF_SOCKET_T == SIZEOF_INT + return (SOCKET_T)_Py_atomic_load_int_relaxed((int *)&s->sock_fd); +#elif SIZEOF_SOCKET_T == SIZEOF_LONG + return (SOCKET_T)_Py_atomic_load_long_relaxed((long *)&s->sock_fd); +#elif SIZEOF_SOCKET_T == SIZEOF_LONG_LONG + return (SOCKET_T)_Py_atomic_load_llong_relaxed((long long *)&s->sock_fd); +#else + #error "Unsupported SIZEOF_SOCKET_T" +#endif +#else + return s->sock_fd; +#endif +} + static inline socket_state * get_module_state(PyObject *mod) { @@ -736,10 +773,10 @@ internal_setblocking(PySocketSockObject *s, int block) #ifndef MS_WINDOWS #if (defined(HAVE_SYS_IOCTL_H) && defined(FIONBIO)) block = !block; - if (ioctl(s->sock_fd, FIONBIO, (unsigned int *)&block) == -1) + if (ioctl(get_sock_fd(s), FIONBIO, (unsigned int *)&block) == -1) goto done; #else - delay_flag = fcntl(s->sock_fd, F_GETFL, 0); + delay_flag = fcntl(get_sock_fd(s), F_GETFL, 0); if (delay_flag == -1) goto done; if (block) @@ -747,12 +784,12 @@ internal_setblocking(PySocketSockObject *s, int block) else new_delay_flag = delay_flag | O_NONBLOCK; if (new_delay_flag != delay_flag) - if (fcntl(s->sock_fd, F_SETFL, new_delay_flag) == -1) + if (fcntl(get_sock_fd(s), F_SETFL, new_delay_flag) == -1) goto done; #endif #else /* MS_WINDOWS */ arg = !block; - if (ioctlsocket(s->sock_fd, FIONBIO, &arg) != 0) + if (ioctlsocket(get_sock_fd(s), FIONBIO, &arg) != 0) goto done; #endif /* MS_WINDOWS */ @@ -792,13 +829,13 @@ internal_select(PySocketSockObject *s, int writing, PyTime_t interval, assert(!(connect && !writing)); /* Guard against closed socket */ - if (s->sock_fd == INVALID_SOCKET) + if (get_sock_fd(s) == INVALID_SOCKET) return 0; /* Prefer poll, if available, since you can poll() any fd * which can't be done with select(). */ #ifdef HAVE_POLL - pollfd.fd = s->sock_fd; + pollfd.fd = get_sock_fd(s); pollfd.events = writing ? POLLOUT : POLLIN; if (connect) { /* On Windows, the socket becomes writable on connection success, @@ -838,23 +875,23 @@ internal_select(PySocketSockObject *s, int writing, PyTime_t interval, tvp = NULL; FD_ZERO(&fds); - FD_SET(s->sock_fd, &fds); + FD_SET(get_sock_fd(s), &fds); FD_ZERO(&efds); if (connect) { /* On Windows, the socket becomes writable on connection success, but a connection failure is notified as an error. On POSIX, the socket becomes writable on connection success or on connection failure. */ - FD_SET(s->sock_fd, &efds); + FD_SET(get_sock_fd(s), &efds); } /* See if the socket is ready */ Py_BEGIN_ALLOW_THREADS; if (writing) - n = select(Py_SAFE_DOWNCAST(s->sock_fd+1, SOCKET_T, int), + n = select(Py_SAFE_DOWNCAST(get_sock_fd(s)+1, SOCKET_T, int), NULL, &fds, &efds, tvp); else - n = select(Py_SAFE_DOWNCAST(s->sock_fd+1, SOCKET_T, int), + n = select(Py_SAFE_DOWNCAST(get_sock_fd(s)+1, SOCKET_T, int), &fds, NULL, &efds, tvp); Py_END_ALLOW_THREADS; #endif @@ -1030,7 +1067,7 @@ static int init_sockobject(socket_state *state, PySocketSockObject *s, SOCKET_T fd, int family, int type, int proto) { - s->sock_fd = fd; + set_sock_fd(s, fd); s->sock_family = family; s->sock_type = type; @@ -2127,7 +2164,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } strncpy(ifr.ifr_name, interfaceName, sizeof(ifr.ifr_name)); ifr.ifr_name[(sizeof(ifr.ifr_name))-1] = '\0'; - if (ioctl(s->sock_fd, SIOCGIFINDEX, &ifr) < 0) { + if (ioctl(get_sock_fd(s), SIOCGIFINDEX, &ifr) < 0) { s->errorhandler(); PyBuffer_Release(&haddr); return 0; @@ -2252,7 +2289,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } else if ((size_t)len < sizeof(ifr.ifr_name)) { strncpy(ifr.ifr_name, PyBytes_AS_STRING(interfaceName), sizeof(ifr.ifr_name)); ifr.ifr_name[(sizeof(ifr.ifr_name))-1] = '\0'; - if (ioctl(s->sock_fd, SIOCGIFINDEX, &ifr) < 0) { + if (ioctl(get_sock_fd(s), SIOCGIFINDEX, &ifr) < 0) { s->errorhandler(); Py_DECREF(interfaceName); return 0; @@ -2296,7 +2333,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } else if ((size_t)len < sizeof(ifr.ifr_name)) { strncpy(ifr.ifr_name, PyBytes_AS_STRING(interfaceName), sizeof(ifr.ifr_name)); ifr.ifr_name[(sizeof(ifr.ifr_name))-1] = '\0'; - if (ioctl(s->sock_fd, SIOCGIFINDEX, &ifr) < 0) { + if (ioctl(get_sock_fd(s), SIOCGIFINDEX, &ifr) < 0) { s->errorhandler(); Py_DECREF(interfaceName); return 0; @@ -2344,7 +2381,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } else if ((size_t)len < sizeof(ifr.ifr_name)) { strncpy(ifr.ifr_name, PyBytes_AS_STRING(interfaceName), sizeof(ifr.ifr_name)); ifr.ifr_name[(sizeof(ifr.ifr_name))-1] = '\0'; - if (ioctl(s->sock_fd, SIOCGIFINDEX, &ifr) < 0) { + if (ioctl(get_sock_fd(s), SIOCGIFINDEX, &ifr) < 0) { s->errorhandler(); Py_DECREF(interfaceName); return 0; @@ -2403,7 +2440,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, sizeof(info.ctl_name)); Py_DECREF(ctl_name); - if (ioctl(s->sock_fd, CTLIOCGINFO, &info)) { + if (ioctl(get_sock_fd(s), CTLIOCGINFO, &info)) { PyErr_SetString(PyExc_OSError, "cannot find kernel control with provided name"); return 0; @@ -2869,7 +2906,7 @@ sock_accept_impl(PySocketSockObject *s, void *data) #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) socket_state *state = s->state; if (state->accept4_works != 0) { - ctx->result = accept4(s->sock_fd, addr, paddrlen, + ctx->result = accept4(get_sock_fd(s), addr, paddrlen, SOCK_CLOEXEC); if (ctx->result == INVALID_SOCKET && state->accept4_works == -1) { /* On Linux older than 2.6.28, accept4() fails with ENOSYS */ @@ -2877,9 +2914,9 @@ sock_accept_impl(PySocketSockObject *s, void *data) } } if (state->accept4_works == 0) - ctx->result = accept(s->sock_fd, addr, paddrlen); + ctx->result = accept(get_sock_fd(s), addr, paddrlen); #else - ctx->result = accept(s->sock_fd, addr, paddrlen); + ctx->result = accept(get_sock_fd(s), addr, paddrlen); #endif #ifdef MS_WINDOWS @@ -2946,7 +2983,7 @@ sock_accept(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) goto finally; } - addr = makesockaddr(s->sock_fd, SAS2SA(&addrbuf), + addr = makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), addrlen, s->sock_proto); if (addr == NULL) goto finally; @@ -3158,7 +3195,7 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) if (PyArg_ParseTuple(args, "iiK:setsockopt", &level, &optname, &vflag)) { // level should always be set to AF_VSOCK - res = setsockopt(s->sock_fd, level, optname, + res = setsockopt(get_sock_fd(s), level, optname, (void*)&vflag, sizeof vflag); goto done; } @@ -3172,7 +3209,7 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) #ifdef MS_WINDOWS if (optname == SIO_TCP_SET_ACK_FREQUENCY) { int dummy; - res = WSAIoctl(s->sock_fd, SIO_TCP_SET_ACK_FREQUENCY, &flag, + res = WSAIoctl(get_sock_fd(s), SIO_TCP_SET_ACK_FREQUENCY, &flag, sizeof(flag), NULL, 0, &dummy, NULL, NULL); if (res >= 0) { s->quickack = flag; @@ -3180,7 +3217,7 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) goto done; } #endif - res = setsockopt(s->sock_fd, level, optname, + res = setsockopt(get_sock_fd(s), level, optname, (char*)&flag, sizeof flag); goto done; } @@ -3190,7 +3227,7 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) if (PyArg_ParseTuple(args, "iiO!I:setsockopt", &level, &optname, Py_TYPE(Py_None), &none, &optlen)) { assert(sizeof(socklen_t) >= sizeof(unsigned int)); - res = setsockopt(s->sock_fd, level, optname, + res = setsockopt(get_sock_fd(s), level, optname, NULL, (socklen_t)optlen); goto done; } @@ -3209,10 +3246,10 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) INT_MAX); return NULL; } - res = setsockopt(s->sock_fd, level, optname, + res = setsockopt(get_sock_fd(s), level, optname, optval.buf, (int)optval.len); #else - res = setsockopt(s->sock_fd, level, optname, optval.buf, optval.len); + res = setsockopt(get_sock_fd(s), level, optname, optval.buf, optval.len); #endif PyBuffer_Release(&optval); @@ -3259,7 +3296,7 @@ sock_getsockopt(PySocketSockObject *s, PyObject *args) if (s->sock_family == AF_VSOCK) { uint64_t vflag = 0; // Must be set width of 64 bits flagsize = sizeof vflag; - res = getsockopt(s->sock_fd, level, optname, + res = getsockopt(get_sock_fd(s), level, optname, (void *)&vflag, &flagsize); if (res < 0) return s->errorhandler(); @@ -3272,7 +3309,7 @@ sock_getsockopt(PySocketSockObject *s, PyObject *args) } #endif flagsize = sizeof flag; - res = getsockopt(s->sock_fd, level, optname, + res = getsockopt(get_sock_fd(s), level, optname, (void *)&flag, &flagsize); if (res < 0) return s->errorhandler(); @@ -3293,7 +3330,7 @@ sock_getsockopt(PySocketSockObject *s, PyObject *args) buf = PyBytes_FromStringAndSize((char *)NULL, buflen); if (buf == NULL) return NULL; - res = getsockopt(s->sock_fd, level, optname, + res = getsockopt(get_sock_fd(s), level, optname, (void *)PyBytes_AS_STRING(buf), &buflen); if (res < 0) { Py_DECREF(buf); @@ -3330,7 +3367,7 @@ sock_bind(PySocketSockObject *s, PyObject *addro) } Py_BEGIN_ALLOW_THREADS - res = bind(s->sock_fd, SAS2SA(&addrbuf), addrlen); + res = bind(get_sock_fd(s), SAS2SA(&addrbuf), addrlen); Py_END_ALLOW_THREADS if (res < 0) return s->errorhandler(); @@ -3367,9 +3404,9 @@ _socket_socket_close_impl(PySocketSockObject *s) SOCKET_T fd; int res; - fd = s->sock_fd; + fd = get_sock_fd(s); if (fd != INVALID_SOCKET) { - s->sock_fd = INVALID_SOCKET; + set_sock_fd(s, INVALID_SOCKET); /* We do not want to retry upon EINTR: see http://lwn.net/Articles/576478/ and @@ -3390,8 +3427,8 @@ _socket_socket_close_impl(PySocketSockObject *s) static PyObject * sock_detach(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) { - SOCKET_T fd = s->sock_fd; - s->sock_fd = INVALID_SOCKET; + SOCKET_T fd = get_sock_fd(s); + set_sock_fd(s, INVALID_SOCKET); return PyLong_FromSocket_t(fd); } @@ -3409,7 +3446,7 @@ sock_connect_impl(PySocketSockObject *s, void* Py_UNUSED(data)) int err; socklen_t size = sizeof err; - if (getsockopt(s->sock_fd, SOL_SOCKET, SO_ERROR, (void *)&err, &size)) { + if (getsockopt(get_sock_fd(s), SOL_SOCKET, SO_ERROR, (void *)&err, &size)) { /* getsockopt() failed */ return 0; } @@ -3443,7 +3480,7 @@ internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen, int res, err, wait_connect; Py_BEGIN_ALLOW_THREADS - res = connect(s->sock_fd, addr, addrlen); + res = connect(get_sock_fd(s), addr, addrlen); Py_END_ALLOW_THREADS if (!res) { @@ -3573,7 +3610,7 @@ instead of raising an exception when an error occurs."); static PyObject * sock_fileno(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) { - return PyLong_FromSocket_t(s->sock_fd); + return PyLong_FromSocket_t(get_sock_fd(s)); } PyDoc_STRVAR(fileno_doc, @@ -3596,11 +3633,11 @@ sock_getsockname(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) return NULL; memset(&addrbuf, 0, addrlen); Py_BEGIN_ALLOW_THREADS - res = getsockname(s->sock_fd, SAS2SA(&addrbuf), &addrlen); + res = getsockname(get_sock_fd(s), SAS2SA(&addrbuf), &addrlen); Py_END_ALLOW_THREADS if (res < 0) return s->errorhandler(); - return makesockaddr(s->sock_fd, SAS2SA(&addrbuf), addrlen, + return makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), addrlen, s->sock_proto); } @@ -3628,11 +3665,11 @@ sock_getpeername(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) return NULL; memset(&addrbuf, 0, addrlen); Py_BEGIN_ALLOW_THREADS - res = getpeername(s->sock_fd, SAS2SA(&addrbuf), &addrlen); + res = getpeername(get_sock_fd(s), SAS2SA(&addrbuf), &addrlen); Py_END_ALLOW_THREADS if (res < 0) return s->errorhandler(); - return makesockaddr(s->sock_fd, SAS2SA(&addrbuf), addrlen, + return makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), addrlen, s->sock_proto); } @@ -3664,7 +3701,7 @@ sock_listen(PySocketSockObject *s, PyObject *args) * (which doesn't make sense anyway) we force a minimum value of 0. */ if (backlog < 0) backlog = 0; - res = listen(s->sock_fd, backlog); + res = listen(get_sock_fd(s), backlog); Py_END_ALLOW_THREADS if (res < 0) return s->errorhandler(); @@ -3695,9 +3732,9 @@ sock_recv_impl(PySocketSockObject *s, void *data) #ifdef MS_WINDOWS if (ctx->len > INT_MAX) ctx->len = INT_MAX; - ctx->result = recv(s->sock_fd, ctx->cbuf, (int)ctx->len, ctx->flags); + ctx->result = recv(get_sock_fd(s), ctx->cbuf, (int)ctx->len, ctx->flags); #else - ctx->result = recv(s->sock_fd, ctx->cbuf, ctx->len, ctx->flags); + ctx->result = recv(get_sock_fd(s), ctx->cbuf, ctx->len, ctx->flags); #endif return (ctx->result >= 0); } @@ -3866,10 +3903,10 @@ sock_recvfrom_impl(PySocketSockObject *s, void *data) #ifdef MS_WINDOWS if (ctx->len > INT_MAX) ctx->len = INT_MAX; - ctx->result = recvfrom(s->sock_fd, ctx->cbuf, (int)ctx->len, ctx->flags, + ctx->result = recvfrom(get_sock_fd(s), ctx->cbuf, (int)ctx->len, ctx->flags, SAS2SA(ctx->addrbuf), ctx->addrlen); #else - ctx->result = recvfrom(s->sock_fd, ctx->cbuf, ctx->len, ctx->flags, + ctx->result = recvfrom(get_sock_fd(s), ctx->cbuf, ctx->len, ctx->flags, SAS2SA(ctx->addrbuf), ctx->addrlen); #endif return (ctx->result >= 0); @@ -3913,7 +3950,7 @@ sock_recvfrom_guts(PySocketSockObject *s, char* cbuf, Py_ssize_t len, int flags, if (sock_call(s, 0, sock_recvfrom_impl, &ctx) < 0) return -1; - *addr = makesockaddr(s->sock_fd, SAS2SA(&addrbuf), addrlen, + *addr = makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), addrlen, s->sock_proto); if (*addr == NULL) return -1; @@ -4044,7 +4081,7 @@ sock_recvmsg_impl(PySocketSockObject *s, void *data) { struct sock_recvmsg *ctx = data; - ctx->result = recvmsg(s->sock_fd, ctx->msg, ctx->flags); + ctx->result = recvmsg(get_sock_fd(s), ctx->msg, ctx->flags); return (ctx->result >= 0); } @@ -4153,7 +4190,7 @@ sock_recvmsg_guts(PySocketSockObject *s, struct iovec *iov, int iovlen, (*makeval)(ctx.result, makeval_data), cmsg_list, (int)msg.msg_flags, - makesockaddr(s->sock_fd, SAS2SA(&addrbuf), + makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), ((msg.msg_namelen > addrbuflen) ? addrbuflen : msg.msg_namelen), s->sock_proto)); @@ -4372,9 +4409,9 @@ sock_send_impl(PySocketSockObject *s, void *data) #ifdef MS_WINDOWS if (ctx->len > INT_MAX) ctx->len = INT_MAX; - ctx->result = send(s->sock_fd, ctx->buf, (int)ctx->len, ctx->flags); + ctx->result = send(get_sock_fd(s), ctx->buf, (int)ctx->len, ctx->flags); #else - ctx->result = send(s->sock_fd, ctx->buf, ctx->len, ctx->flags); + ctx->result = send(get_sock_fd(s), ctx->buf, ctx->len, ctx->flags); #endif return (ctx->result >= 0); } @@ -4511,10 +4548,10 @@ sock_sendto_impl(PySocketSockObject *s, void *data) #ifdef MS_WINDOWS if (ctx->len > INT_MAX) ctx->len = INT_MAX; - ctx->result = sendto(s->sock_fd, ctx->buf, (int)ctx->len, ctx->flags, + ctx->result = sendto(get_sock_fd(s), ctx->buf, (int)ctx->len, ctx->flags, SAS2SA(ctx->addrbuf), ctx->addrlen); #else - ctx->result = sendto(s->sock_fd, ctx->buf, ctx->len, ctx->flags, + ctx->result = sendto(get_sock_fd(s), ctx->buf, ctx->len, ctx->flags, SAS2SA(ctx->addrbuf), ctx->addrlen); #endif return (ctx->result >= 0); @@ -4660,7 +4697,7 @@ sock_sendmsg_impl(PySocketSockObject *s, void *data) { struct sock_sendmsg *ctx = data; - ctx->result = sendmsg(s->sock_fd, ctx->msg, ctx->flags); + ctx->result = sendmsg(get_sock_fd(s), ctx->msg, ctx->flags); return (ctx->result >= 0); } @@ -5048,7 +5085,7 @@ sock_shutdown(PySocketSockObject *s, PyObject *arg) if (how == -1 && PyErr_Occurred()) return NULL; Py_BEGIN_ALLOW_THREADS - res = shutdown(s->sock_fd, how); + res = shutdown(get_sock_fd(s), how); Py_END_ALLOW_THREADS if (res < 0) return s->errorhandler(); @@ -5078,7 +5115,7 @@ sock_ioctl(PySocketSockObject *s, PyObject *arg) unsigned int option = RCVALL_ON; if (!PyArg_ParseTuple(arg, "kI:ioctl", &cmd, &option)) return NULL; - if (WSAIoctl(s->sock_fd, cmd, &option, sizeof(option), + if (WSAIoctl(get_sock_fd(s), cmd, &option, sizeof(option), NULL, 0, &recv, NULL, NULL) == SOCKET_ERROR) { return set_error(); } @@ -5088,7 +5125,7 @@ sock_ioctl(PySocketSockObject *s, PyObject *arg) if (!PyArg_ParseTuple(arg, "k(kkk):ioctl", &cmd, &ka.onoff, &ka.keepalivetime, &ka.keepaliveinterval)) return NULL; - if (WSAIoctl(s->sock_fd, cmd, &ka, sizeof(ka), + if (WSAIoctl(get_sock_fd(s), cmd, &ka, sizeof(ka), NULL, 0, &recv, NULL, NULL) == SOCKET_ERROR) { return set_error(); } @@ -5098,7 +5135,7 @@ sock_ioctl(PySocketSockObject *s, PyObject *arg) unsigned int option; if (!PyArg_ParseTuple(arg, "kI:ioctl", &cmd, &option)) return NULL; - if (WSAIoctl(s->sock_fd, cmd, &option, sizeof(option), + if (WSAIoctl(get_sock_fd(s), cmd, &option, sizeof(option), NULL, 0, &recv, NULL, NULL) == SOCKET_ERROR) { return set_error(); } @@ -5130,7 +5167,7 @@ sock_share(PySocketSockObject *s, PyObject *arg) return NULL; Py_BEGIN_ALLOW_THREADS - result = WSADuplicateSocketW(s->sock_fd, processId, &info); + result = WSADuplicateSocketW(get_sock_fd(s), processId, &info); Py_END_ALLOW_THREADS if (result == SOCKET_ERROR) return set_error(); @@ -5264,7 +5301,7 @@ sock_finalize(PySocketSockObject *s) /* Save the current exception, if any. */ PyObject *exc = PyErr_GetRaisedException(); - if (s->sock_fd != INVALID_SOCKET) { + if (get_sock_fd(s) != INVALID_SOCKET) { if (PyErr_ResourceWarning((PyObject *)s, 1, "unclosed %R", s)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) { @@ -5276,8 +5313,8 @@ sock_finalize(PySocketSockObject *s) to allow the logger to call socket methods like socket.getsockname(). If the socket is closed before, socket methods fails with the EBADF error. */ - fd = s->sock_fd; - s->sock_fd = INVALID_SOCKET; + fd = get_sock_fd(s); + set_sock_fd(s, INVALID_SOCKET); /* We do not want to retry upon EINTR: see sock_close() */ Py_BEGIN_ALLOW_THREADS @@ -5314,11 +5351,11 @@ sock_repr(PySocketSockObject *s) { long sock_fd; /* On Windows, this test is needed because SOCKET_T is unsigned */ - if (s->sock_fd == INVALID_SOCKET) { + if (get_sock_fd(s) == INVALID_SOCKET) { sock_fd = -1; } #if SIZEOF_SOCKET_T > SIZEOF_LONG - else if (s->sock_fd > LONG_MAX) { + else if (get_sock_fd(s) > LONG_MAX) { /* this can occur on Win64, and actually there is a special ugly printf formatter for decimal pointer length integer printing, only bother if necessary*/ @@ -5329,7 +5366,7 @@ sock_repr(PySocketSockObject *s) } #endif else - sock_fd = (long)s->sock_fd; + sock_fd = (long)get_sock_fd(s); return PyUnicode_FromFormat( "", sock_fd, s->sock_family, From b2ac70a62ad1be8e037ce45ccf5f1b753ea5e64b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 31 Dec 2024 10:02:58 +0200 Subject: [PATCH 321/427] gh-88834: Unify the instance check for typing.Union and types.UnionType (GH-128363) Union now uses the instance checks against its parameters instead of the subclass checks. --- Lib/test/test_typing.py | 75 ++++++++++++++++++- Lib/typing.py | 6 +- ...4-12-30-20-48-28.gh-issue-88834.RIvgwc.rst | 3 + 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-30-20-48-28.gh-issue-88834.RIvgwc.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index aa42beca5f9256..ef3cfc9517085e 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -122,7 +122,7 @@ class Sub(Any): pass def test_errors(self): with self.assertRaises(TypeError): - issubclass(42, Any) + isinstance(42, Any) with self.assertRaises(TypeError): Any[int] # Any is not a generic type. @@ -137,6 +137,9 @@ class Something: pass class MockSomething(Something, Mock): pass self.assertTrue(issubclass(MockSomething, Any)) + self.assertTrue(issubclass(MockSomething, MockSomething)) + self.assertTrue(issubclass(MockSomething, Something)) + self.assertTrue(issubclass(MockSomething, Mock)) ms = MockSomething() self.assertIsInstance(ms, MockSomething) self.assertIsInstance(ms, Something) @@ -2010,13 +2013,81 @@ def test_basics(self): u = Union[int, float] self.assertNotEqual(u, Union) - def test_subclass_error(self): + def test_union_isinstance(self): + self.assertTrue(isinstance(42, Union[int, str])) + self.assertTrue(isinstance('abc', Union[int, str])) + self.assertFalse(isinstance(3.14, Union[int, str])) + self.assertTrue(isinstance(42, Union[int, list[int]])) + self.assertTrue(isinstance(42, Union[int, Any])) + + def test_union_isinstance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Union[str, list[int]]) + with self.assertRaises(TypeError): + isinstance(42, Union[list[int], int]) + with self.assertRaises(TypeError): + isinstance(42, Union[list[int], str]) + with self.assertRaises(TypeError): + isinstance(42, Union[str, Any]) + with self.assertRaises(TypeError): + isinstance(42, Union[Any, int]) + with self.assertRaises(TypeError): + isinstance(42, Union[Any, str]) + + def test_optional_isinstance(self): + self.assertTrue(isinstance(42, Optional[int])) + self.assertTrue(isinstance(None, Optional[int])) + self.assertFalse(isinstance('abc', Optional[int])) + + def test_optional_isinstance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Optional[list[int]]) + with self.assertRaises(TypeError): + isinstance(None, Optional[list[int]]) + with self.assertRaises(TypeError): + isinstance(42, Optional[Any]) + with self.assertRaises(TypeError): + isinstance(None, Optional[Any]) + + def test_union_issubclass(self): + self.assertTrue(issubclass(int, Union[int, str])) + self.assertTrue(issubclass(str, Union[int, str])) + self.assertFalse(issubclass(float, Union[int, str])) + self.assertTrue(issubclass(int, Union[int, list[int]])) + self.assertTrue(issubclass(int, Union[int, Any])) + self.assertFalse(issubclass(int, Union[str, Any])) + self.assertTrue(issubclass(int, Union[Any, int])) + self.assertFalse(issubclass(int, Union[Any, str])) + + def test_union_issubclass_type_error(self): with self.assertRaises(TypeError): issubclass(int, Union) with self.assertRaises(TypeError): issubclass(Union, int) with self.assertRaises(TypeError): issubclass(Union[int, str], int) + with self.assertRaises(TypeError): + issubclass(int, Union[str, list[int]]) + with self.assertRaises(TypeError): + issubclass(int, Union[list[int], int]) + with self.assertRaises(TypeError): + issubclass(int, Union[list[int], str]) + + def test_optional_issubclass(self): + self.assertTrue(issubclass(int, Optional[int])) + self.assertTrue(issubclass(type(None), Optional[int])) + self.assertFalse(issubclass(str, Optional[int])) + self.assertTrue(issubclass(Any, Optional[Any])) + self.assertTrue(issubclass(type(None), Optional[Any])) + self.assertFalse(issubclass(int, Optional[Any])) + + def test_optional_issubclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(list[int], Optional[list[int]]) + with self.assertRaises(TypeError): + issubclass(type(None), Optional[list[int]]) + with self.assertRaises(TypeError): + issubclass(int, Optional[list[int]]) def test_union_any(self): u = Union[Any] diff --git a/Lib/typing.py b/Lib/typing.py index 5f3aacd877221c..e69b485422cbd2 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1733,12 +1733,16 @@ def __repr__(self): return super().__repr__() def __instancecheck__(self, obj): - return self.__subclasscheck__(type(obj)) + for arg in self.__args__: + if isinstance(obj, arg): + return True + return False def __subclasscheck__(self, cls): for arg in self.__args__: if issubclass(cls, arg): return True + return False def __reduce__(self): func, (origin, args) = super().__reduce__() diff --git a/Misc/NEWS.d/next/Library/2024-12-30-20-48-28.gh-issue-88834.RIvgwc.rst b/Misc/NEWS.d/next/Library/2024-12-30-20-48-28.gh-issue-88834.RIvgwc.rst new file mode 100644 index 00000000000000..ca43f914880ba3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-30-20-48-28.gh-issue-88834.RIvgwc.rst @@ -0,0 +1,3 @@ +Unify the instance check for :class:`typing.Union` and +:class:`types.UnionType`: :class:`!Union` now uses the instance checks +against its parameters instead of the subclass checks. From e389d6c650ddacb55b08b657f1e4e9b1330f3455 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 31 Dec 2024 19:10:06 +0530 Subject: [PATCH 322/427] gh-128277: make globals variables thread safe in socket module (#128286) --- Modules/socketmodule.c | 56 +++++++++------------ Python/fileutils.c | 6 +-- Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 + 3 files changed, 28 insertions(+), 36 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 1f0cbf6abb184c..efc8be3ab592ba 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -550,20 +550,20 @@ typedef struct _socket_state { /* Default timeout for new sockets */ PyTime_t defaulttimeout; +} socket_state; #if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - /* accept4() is available on Linux 2.6.28+ and glibc 2.10 */ - int accept4_works; +/* accept4() is available on Linux 2.6.28+ and glibc 2.10 */ +static int accept4_works = -1; #endif #endif #ifdef SOCK_CLOEXEC - /* socket() and socketpair() fail with EINVAL on Linux kernel older - * than 2.6.27 if SOCK_CLOEXEC flag is set in the socket type. */ - int sock_cloexec_works; +/* socket() and socketpair() fail with EINVAL on Linux kernel older + * than 2.6.27 if SOCK_CLOEXEC flag is set in the socket type. */ +static int sock_cloexec_works = -1; #endif -} socket_state; static inline void set_sock_fd(PySocketSockObject *s, SOCKET_T fd) @@ -2904,16 +2904,15 @@ sock_accept_impl(PySocketSockObject *s, void *data) #endif #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - socket_state *state = s->state; - if (state->accept4_works != 0) { + if (_Py_atomic_load_int_relaxed(&accept4_works) != 0) { ctx->result = accept4(get_sock_fd(s), addr, paddrlen, SOCK_CLOEXEC); - if (ctx->result == INVALID_SOCKET && state->accept4_works == -1) { + if (ctx->result == INVALID_SOCKET && _Py_atomic_load_int_relaxed(&accept4_works) == -1) { /* On Linux older than 2.6.28, accept4() fails with ENOSYS */ - state->accept4_works = (errno != ENOSYS); + _Py_atomic_store_int_relaxed(&accept4_works, errno != ENOSYS); } } - if (state->accept4_works == 0) + if (_Py_atomic_load_int_relaxed(&accept4_works) == 0) ctx->result = accept(get_sock_fd(s), addr, paddrlen); #else ctx->result = accept(get_sock_fd(s), addr, paddrlen); @@ -2966,8 +2965,7 @@ sock_accept(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) #else #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - socket_state *state = s->state; - if (!state->accept4_works) + if (!_Py_atomic_load_int_relaxed(&accept4_works)) #endif { if (_Py_set_inheritable(newfd, 0, NULL) < 0) { @@ -5428,7 +5426,7 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, #ifndef MS_WINDOWS #ifdef SOCK_CLOEXEC - int *atomic_flag_works = &state->sock_cloexec_works; + int *atomic_flag_works = &sock_cloexec_works; #else int *atomic_flag_works = NULL; #endif @@ -5583,15 +5581,16 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, /* UNIX */ Py_BEGIN_ALLOW_THREADS #ifdef SOCK_CLOEXEC - if (state->sock_cloexec_works != 0) { + if (_Py_atomic_load_int_relaxed(&sock_cloexec_works) != 0) { fd = socket(family, type | SOCK_CLOEXEC, proto); - if (state->sock_cloexec_works == -1) { + if (_Py_atomic_load_int_relaxed(&sock_cloexec_works) == -1) { if (fd >= 0) { - state->sock_cloexec_works = 1; + _Py_atomic_store_int_relaxed(&sock_cloexec_works, 1); } + else if (errno == EINVAL) { /* Linux older than 2.6.27 does not support SOCK_CLOEXEC */ - state->sock_cloexec_works = 0; + _Py_atomic_store_int_relaxed(&sock_cloexec_works, 0); fd = socket(family, type, proto); } } @@ -6332,7 +6331,7 @@ socket_socketpair(PyObject *self, PyObject *args) PyObject *res = NULL; socket_state *state = get_module_state(self); #ifdef SOCK_CLOEXEC - int *atomic_flag_works = &state->sock_cloexec_works; + int *atomic_flag_works = &sock_cloexec_works; #else int *atomic_flag_works = NULL; #endif @@ -6350,15 +6349,15 @@ socket_socketpair(PyObject *self, PyObject *args) /* Create a pair of socket fds */ Py_BEGIN_ALLOW_THREADS #ifdef SOCK_CLOEXEC - if (state->sock_cloexec_works != 0) { + if (_Py_atomic_load_int_relaxed(&sock_cloexec_works) != 0) { ret = socketpair(family, type | SOCK_CLOEXEC, proto, sv); - if (state->sock_cloexec_works == -1) { + if (_Py_atomic_load_int_relaxed(&sock_cloexec_works) == -1) { if (ret >= 0) { - state->sock_cloexec_works = 1; + _Py_atomic_store_int_relaxed(&sock_cloexec_works, 1); } else if (errno == EINVAL) { /* Linux older than 2.6.27 does not support SOCK_CLOEXEC */ - state->sock_cloexec_works = 0; + _Py_atomic_store_int_relaxed(&sock_cloexec_works, 0); ret = socketpair(family, type, proto, sv); } } @@ -7466,17 +7465,8 @@ socket_exec(PyObject *m) } socket_state *state = get_module_state(m); - state->defaulttimeout = _PYTIME_FROMSECONDS(-1); - -#if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) -#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - state->accept4_works = -1; -#endif -#endif -#ifdef SOCK_CLOEXEC - state->sock_cloexec_works = -1; -#endif + _Py_atomic_store_int64_relaxed(&state->defaulttimeout, _PYTIME_FROMSECONDS(-1)); #define ADD_EXC(MOD, NAME, VAR, BASE) do { \ VAR = PyErr_NewException("socket." NAME, BASE, NULL); \ diff --git a/Python/fileutils.c b/Python/fileutils.c index 9529b14d377c60..81276651f6df44 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1468,14 +1468,14 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) assert(!(atomic_flag_works != NULL && inheritable)); if (atomic_flag_works != NULL && !inheritable) { - if (*atomic_flag_works == -1) { + if (_Py_atomic_load_int_relaxed(atomic_flag_works) == -1) { int isInheritable = get_inheritable(fd, raise); if (isInheritable == -1) return -1; - *atomic_flag_works = !isInheritable; + _Py_atomic_store_int_relaxed(atomic_flag_works, !isInheritable); } - if (*atomic_flag_works) + if (_Py_atomic_load_int_relaxed(atomic_flag_works)) return 0; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index a1ec1927eb56df..a74779803228c2 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -444,3 +444,5 @@ Modules/readline.c - completed_input_string - Modules/rotatingtree.c - random_stream - Modules/rotatingtree.c - random_value - Modules/rotatingtree.c - random_mutex - +Modules/socketmodule.c - accept4_works - +Modules/socketmodule.c - sock_cloexec_works - From c5438fdf4706a70bdd19338edc000dacffff6837 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Tue, 31 Dec 2024 21:22:33 +0000 Subject: [PATCH 323/427] Reword `about.rst` to not limit Sphinx (#128325) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/about.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Doc/about.rst b/Doc/about.rst index 5e6160ff2700ed..8f635d7f743a98 100644 --- a/Doc/about.rst +++ b/Doc/about.rst @@ -1,10 +1,11 @@ -===================== -About these documents -===================== +======================== +About this documentation +======================== -These documents are generated from `reStructuredText`_ sources by `Sphinx`_, a -document processor specifically written for the Python documentation. +Python's documentation is generated from `reStructuredText`_ sources +using `Sphinx`_, a documentation generator originally created for Python +and now maintained as an independent project. .. _reStructuredText: https://docutils.sourceforge.io/rst.html .. _Sphinx: https://www.sphinx-doc.org/ @@ -20,14 +21,14 @@ volunteers are always welcome! Many thanks go to: * Fred L. Drake, Jr., the creator of the original Python documentation toolset - and writer of much of the content; + and author of much of the content; * the `Docutils `_ project for creating reStructuredText and the Docutils suite; * Fredrik Lundh for his Alternative Python Reference project from which Sphinx got many good ideas. -Contributors to the Python Documentation +Contributors to the Python documentation ---------------------------------------- Many people have contributed to the Python language, the Python standard From d903b17499b1a3bfb3ea848f6a1b6da02eac3328 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 1 Jan 2025 13:36:47 +0200 Subject: [PATCH 324/427] gh-121676: Raise a ``DeprecationWarning`` if the Python implementation of ``functools.reduce`` is called with `function` or `sequence` as a keyword args (#121677) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python implementation of `functools` allows calling `reduce` with `function` or `sequence` as keyword args. This doesn't match behavior of our C accelerator and our documentation for `functools.reduce` states that `function`and `sequence` are positional-only arguments. Now calling a Python implementation of `functools.reduce` with `function` or `sequence` as keyword args would raise a `DeprecationWarning` and is planned to be prohibited in Python 3.16. Co-authored-by: Victor Stinner Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/deprecations/pending-removal-in-3.16.rst | 5 +++ Doc/whatsnew/3.14.rst | 5 +++ Lib/functools.py | 33 ++++++++++++++++--- Lib/test/test_functools.py | 6 ++++ ...-07-13-13-25-31.gh-issue-121676.KDLS11.rst | 3 ++ 5 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-07-13-13-25-31.gh-issue-121676.KDLS11.rst diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index f2b818f14c63cd..d093deb648baf7 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -79,3 +79,8 @@ Pending removal in Python 3.16 * The undocumented and unused :attr:`!TarFile.tarfile` attribute has been deprecated since Python 3.13. + +* :mod:`functools`: + + * Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments has been deprecated since Python 3.14. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 63fa21e17bc834..4b3f1b2e8eed42 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -744,6 +744,11 @@ Deprecated as a single positional argument. (Contributed by Serhiy Storchaka in :gh:`109218`.) +* :mod:`functools`: + Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments is now deprecated. + (Contributed by Kirill Podoprigora in :gh:`121676`.) + * :mod:`os`: :term:`Soft deprecate ` :func:`os.popen` and :func:`os.spawn* ` functions. They should no longer be used to diff --git a/Lib/functools.py b/Lib/functools.py index 786b8aedfd77f5..fd33f0ae479ddc 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -264,11 +264,6 @@ def reduce(function, sequence, initial=_initial_missing): return value -try: - from _functools import reduce -except ImportError: - pass - ################################################################################ ### partial() argument application @@ -1124,3 +1119,31 @@ def __get__(self, instance, owner=None): return val __class_getitem__ = classmethod(GenericAlias) + +def _warn_python_reduce_kwargs(py_reduce): + @wraps(py_reduce) + def wrapper(*args, **kwargs): + if 'function' in kwargs or 'sequence' in kwargs: + import os + import warnings + warnings.warn( + 'Calling functools.reduce with keyword arguments ' + '"function" or "sequence" ' + 'is deprecated in Python 3.14 and will be ' + 'forbidden in Python 3.16.', + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) + return py_reduce(*args, **kwargs) + return wrapper + +reduce = _warn_python_reduce_kwargs(reduce) +del _warn_python_reduce_kwargs + +# The import of the C accelerated version of reduce() has been moved +# here due to gh-121676. In Python 3.16, _warn_python_reduce_kwargs() +# should be removed and the import block should be moved back right +# after the definition of reduce(). +try: + from _functools import reduce +except ImportError: + pass diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 4a0252cb637a52..32224866082824 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1045,6 +1045,12 @@ class TestReduceC(TestReduce, unittest.TestCase): class TestReducePy(TestReduce, unittest.TestCase): reduce = staticmethod(py_functools.reduce) + def test_reduce_with_kwargs(self): + with self.assertWarns(DeprecationWarning): + self.reduce(function=lambda x, y: x + y, sequence=[1, 2, 3, 4, 5], initial=1) + with self.assertWarns(DeprecationWarning): + self.reduce(lambda x, y: x + y, sequence=[1, 2, 3, 4, 5], initial=1) + class TestCmpToKey: diff --git a/Misc/NEWS.d/next/Library/2024-07-13-13-25-31.gh-issue-121676.KDLS11.rst b/Misc/NEWS.d/next/Library/2024-07-13-13-25-31.gh-issue-121676.KDLS11.rst new file mode 100644 index 00000000000000..be589b727a1968 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-13-13-25-31.gh-issue-121676.KDLS11.rst @@ -0,0 +1,3 @@ +Deprecate calling the Python implementation of :meth:`functools.reduce` +with a ``function`` or ``sequence`` as a :term:`keyword argument`. +This will be forbidden in Python 3.16 in order to match the C implementation. From bb9d955e16c5578bdbc72750fbbffc8313559109 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 1 Jan 2025 18:00:47 +0530 Subject: [PATCH 325/427] gh-128277: remove unnecessary critical section from `socket.close` (#128305) Remove unnecessary critical section from `socket.close` as it now uses relaxed atomics for `sock_fd`. --- Lib/test/test_socket.py | 20 ++++++++++++++++++++ Modules/clinic/socketmodule.c.h | 11 ++--------- Modules/socketmodule.c | 3 +-- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index aac213e36aecf0..faf326d9164e1b 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -7075,6 +7075,26 @@ def close_fds(fds): self.assertEqual(data, str(index).encode()) +class FreeThreadingTests(unittest.TestCase): + + def test_close_detach_race(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def close(): + for _ in range(1000): + s.close() + + def detach(): + for _ in range(1000): + s.detach() + + t1 = threading.Thread(target=close) + t2 = threading.Thread(target=detach) + + with threading_helper.start_threads([t1, t2]): + pass + + def setUpModule(): thread_info = threading_helper.threading_setup() unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index db1a28b86c8773..2152f288a9722f 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -6,7 +6,6 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_socket_socket_close__doc__, @@ -26,13 +25,7 @@ _socket_socket_close_impl(PySocketSockObject *s); static PyObject * _socket_socket_close(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(s); - return_value = _socket_socket_close_impl(s); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _socket_socket_close_impl(s); } static int @@ -287,4 +280,4 @@ _socket_socket_if_nametoindex(PySocketSockObject *self, PyObject *arg) #ifndef _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF #define _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF #endif /* !defined(_SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF) */ -/*[clinic end generated code: output=59c36bb31b05de68 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3e612e8df1c322dd input=a9049054013a1b77]*/ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index efc8be3ab592ba..e70aa304f2f3a3 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -3386,7 +3386,6 @@ sockets the address is a tuple (ifname, proto [,pkttype [,hatype [,addr]]])"); will surely fail. */ /*[clinic input] -@critical_section _socket.socket.close self as s: self(type="PySocketSockObject *") @@ -3397,7 +3396,7 @@ Close the socket. It cannot be used after this call. static PyObject * _socket_socket_close_impl(PySocketSockObject *s) -/*[clinic end generated code: output=038b2418e07f6f6c input=9839a261e05bcb97]*/ +/*[clinic end generated code: output=038b2418e07f6f6c input=dc487e470e55a83c]*/ { SOCKET_T fd; int res; From a327810169982e3782bdefc2247789a71aa79b43 Mon Sep 17 00:00:00 2001 From: "RUANG (James Roy)" Date: Thu, 2 Jan 2025 04:39:28 +0800 Subject: [PATCH 326/427] Fix while statements with non-bool conditions in `_pyrepl` (#127509) Fix non-bool value conditions --- Lib/_pyrepl/simple_interact.py | 2 +- Lib/_pyrepl/unix_console.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 342a4b58bfd0f3..a5033496712a73 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -138,7 +138,7 @@ def maybe_run_command(statement: str) -> bool: return False - while 1: + while True: try: try: sys.stdout.flush() diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 2576b938a34c64..63e8fc24dd7625 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -786,7 +786,7 @@ def __tputs(self, fmt, prog=delayprog): # only if the bps is actually needed (which I'm # betting is pretty unlkely) bps = ratedict.get(self.__svtermstate.ospeed) - while 1: + while True: m = prog.search(fmt) if not m: os.write(self.output_fd, fmt) From c810ed7c8e0a7464d19700ba1c8668a406f1c042 Mon Sep 17 00:00:00 2001 From: qqwqqw689 <114795525+qqwqqw689@users.noreply.github.com> Date: Thu, 2 Jan 2025 06:11:29 +0800 Subject: [PATCH 327/427] gh-126469: remove unnecessary error-checking branch in `lexer.c` (#126473) --- Parser/lexer/lexer.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 8c868593f944c8..dbbb94a407c81d 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -329,11 +329,7 @@ verify_identifier(struct tok_state *tok) return 0; } Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s); - if (invalid < 0) { - Py_DECREF(s); - tok->done = E_ERROR; - return 0; - } + assert(invalid >= 0); assert(PyUnicode_GET_LENGTH(s) > 0); if (invalid < PyUnicode_GET_LENGTH(s)) { Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid); From 60c65184695a3eab766b3bc26fc99f695deb998f Mon Sep 17 00:00:00 2001 From: abkmystery <36216019+abkmystery@users.noreply.github.com> Date: Thu, 2 Jan 2025 01:20:31 -0600 Subject: [PATCH 328/427] gh-128349: Use `.. data::` instead of `.. class::` for pre-defined decimal `Context` objects (#128379) --- Doc/library/decimal.rst | 6 +++--- Doc/whatsnew/3.3.rst | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index c9a3e448cad063..185eaf3f721c72 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1033,7 +1033,7 @@ New contexts can also be created using the :class:`Context` constructor described below. In addition, the module provides three pre-made contexts: -.. class:: BasicContext +.. data:: BasicContext This is a standard context defined by the General Decimal Arithmetic Specification. Precision is set to nine. Rounding is set to @@ -1044,7 +1044,7 @@ described below. In addition, the module provides three pre-made contexts: Because many of the traps are enabled, this context is useful for debugging. -.. class:: ExtendedContext +.. data:: ExtendedContext This is a standard context defined by the General Decimal Arithmetic Specification. Precision is set to nine. Rounding is set to @@ -1057,7 +1057,7 @@ described below. In addition, the module provides three pre-made contexts: presence of conditions that would otherwise halt the program. -.. class:: DefaultContext +.. data:: DefaultContext This context is used by the :class:`Context` constructor as a prototype for new contexts. Changing a field (such a precision) has the effect of changing the diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index f814c4e90d5719..7a8eb47cbdb354 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1147,8 +1147,8 @@ API changes | :const:`MIN_EMIN` | ``-425000000`` | ``-999999999999999999`` | +-------------------+----------------+-------------------------+ -* In the context templates (:class:`~decimal.DefaultContext`, - :class:`~decimal.BasicContext` and :class:`~decimal.ExtendedContext`) +* In the context templates (:const:`~decimal.DefaultContext`, + :const:`~decimal.BasicContext` and :const:`~decimal.ExtendedContext`) the magnitude of :attr:`~decimal.Context.Emax` and :attr:`~decimal.Context.Emin` has changed to ``999999``. From e1baa778f602ede66831eb34b9ef17f21e4d4347 Mon Sep 17 00:00:00 2001 From: Shin-myoung-serp Date: Thu, 2 Jan 2025 18:45:07 +0900 Subject: [PATCH 329/427] =?UTF-8?q?gh-95371:=20Add=20support=20for=20other?= =?UTF-8?q?=20image=20formats(e.g.=20PNG)=20to=20the=20turtle=E2=80=A6=20(?= =?UTF-8?q?#95378)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Oleg Iarygin --- Doc/library/turtle.rst | 24 +++++++++++---- Lib/turtle.py | 30 +++++++++---------- ...2-07-28-12-32-59.gh-issue-95371.F24IFC.rst | 1 + 3 files changed, 34 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-07-28-12-32-59.gh-issue-95371.F24IFC.rst diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 8eb4f8271fcfae..512647f5f6e01f 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -1823,7 +1823,8 @@ Window control .. function:: bgpic(picname=None) - :param picname: a string, name of a gif-file or ``"nopic"``, or ``None`` + :param picname: a string, name of an image file (PNG, GIF, PGM, and PPM) + or ``"nopic"``, or ``None`` Set background image or return name of current backgroundimage. If *picname* is a filename, set the corresponding image as background. If *picname* is @@ -2200,9 +2201,9 @@ Settings and special methods .. function:: register_shape(name, shape=None) addshape(name, shape=None) - There are three different ways to call this function: + There are four different ways to call this function: - (1) *name* is the name of a gif-file and *shape* is ``None``: Install the + (1) *name* is the name of an image file (PNG, GIF, PGM, and PPM) and *shape* is ``None``: Install the corresponding image shape. :: >>> screen.register_shape("turtle.gif") @@ -2211,7 +2212,16 @@ Settings and special methods Image shapes *do not* rotate when turning the turtle, so they do not display the heading of the turtle! - (2) *name* is an arbitrary string and *shape* is a tuple of pairs of + (2) *name* is an arbitrary string and *shape* is the name of an image file (PNG, GIF, PGM, and PPM): Install the + corresponding image shape. :: + + >>> screen.register_shape("turtle", "turtle.gif") + + .. note:: + Image shapes *do not* rotate when turning the turtle, so they do not + display the heading of the turtle! + + (3) *name* is an arbitrary string and *shape* is a tuple of pairs of coordinates: Install the corresponding polygon shape. .. doctest:: @@ -2219,12 +2229,16 @@ Settings and special methods >>> screen.register_shape("triangle", ((5,-3), (0,5), (-5,-3))) - (3) *name* is an arbitrary string and *shape* is a (compound) :class:`Shape` + (4) *name* is an arbitrary string and *shape* is a (compound) :class:`Shape` object: Install the corresponding compound shape. Add a turtle shape to TurtleScreen's shapelist. Only thusly registered shapes can be used by issuing the command ``shape(shapename)``. + .. versionchanged:: next + Added support for PNG, PGM, and PPM image formats. + Both a shape name and an image file name can be specified. + .. function:: turtles() diff --git a/Lib/turtle.py b/Lib/turtle.py index 8a5801f2efe625..1320cfd93fd6db 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -51,7 +51,7 @@ turtle. So the turtles can more easily be used as a visual feedback instrument by the (beginning) programmer. -- Different turtle shapes, gif-images as turtle shapes, user defined +- Different turtle shapes, image files as turtle shapes, user defined and user controllable turtle shapes, among them compound (multicolored) shapes. Turtle shapes can be stretched and tilted, which makes turtles very versatile geometrical objects. @@ -468,7 +468,7 @@ def _blankimage(self): def _image(self, filename): """return an image object containing the - imagedata from a gif-file named filename. + imagedata from an image file named filename. """ return TK.PhotoImage(file=filename, master=self.cv) @@ -872,10 +872,7 @@ def __init__(self, type_, data=None): if isinstance(data, list): data = tuple(data) elif type_ == "image": - if isinstance(data, str): - if data.lower().endswith(".gif") and isfile(data): - data = TurtleScreen._image(data) - # else data assumed to be PhotoImage + assert(isinstance(data, TK.PhotoImage)) elif type_ == "compound": data = [] else: @@ -1100,14 +1097,18 @@ def register_shape(self, name, shape=None): """Adds a turtle shape to TurtleScreen's shapelist. Arguments: - (1) name is the name of a gif-file and shape is None. + (1) name is the name of an image file (PNG, GIF, PGM, and PPM) and shape is None. Installs the corresponding image shape. !! Image-shapes DO NOT rotate when turning the turtle, !! so they do not display the heading of the turtle! - (2) name is an arbitrary string and shape is a tuple + (2) name is an arbitrary string and shape is the name of an image file (PNG, GIF, PGM, and PPM). + Installs the corresponding image shape. + !! Image-shapes DO NOT rotate when turning the turtle, + !! so they do not display the heading of the turtle! + (3) name is an arbitrary string and shape is a tuple of pairs of coordinates. Installs the corresponding polygon shape - (3) name is an arbitrary string and shape is a + (4) name is an arbitrary string and shape is a (compound) Shape object. Installs the corresponding compound shape. To use a shape, you have to issue the command shape(shapename). @@ -1120,12 +1121,9 @@ def register_shape(self, name, shape=None): """ if shape is None: - # image - if name.lower().endswith(".gif"): - shape = Shape("image", self._image(name)) - else: - raise TurtleGraphicsError("Bad arguments for register_shape.\n" - + "Use help(register_shape)" ) + shape = Shape("image", self._image(name)) + elif isinstance(shape, str): + shape = Shape("image", self._image(shape)) elif isinstance(shape, tuple): shape = Shape("polygon", shape) ## else shape assumed to be Shape-instance @@ -1454,7 +1452,7 @@ def bgpic(self, picname=None): """Set background image or return name of current backgroundimage. Optional argument: - picname -- a string, name of a gif-file or "nopic". + picname -- a string, name of an image file (PNG, GIF, PGM, and PPM) or "nopic". If picname is a filename, set the corresponding image as background. If picname is "nopic", delete backgroundimage, if present. diff --git a/Misc/NEWS.d/next/Library/2022-07-28-12-32-59.gh-issue-95371.F24IFC.rst b/Misc/NEWS.d/next/Library/2022-07-28-12-32-59.gh-issue-95371.F24IFC.rst new file mode 100644 index 00000000000000..4a62aaed78b425 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-07-28-12-32-59.gh-issue-95371.F24IFC.rst @@ -0,0 +1 @@ +Added support for other image formats (PNG, PGM, and PPM) to the turtle module. Patch by Shin-myoung-serp. From 8d16919a06a55a50756bf083221a6f6cab43de50 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 2 Jan 2025 13:38:21 +0200 Subject: [PATCH 330/427] gh-123925: Fix building curses on platforms without libncursesw (GH-128405) --- .../next/Build/2025-01-02-11-02-45.gh-issue-123925.TLlyUi.rst | 2 ++ Modules/_cursesmodule.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2025-01-02-11-02-45.gh-issue-123925.TLlyUi.rst diff --git a/Misc/NEWS.d/next/Build/2025-01-02-11-02-45.gh-issue-123925.TLlyUi.rst b/Misc/NEWS.d/next/Build/2025-01-02-11-02-45.gh-issue-123925.TLlyUi.rst new file mode 100644 index 00000000000000..a2a9c6fc7680b5 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-01-02-11-02-45.gh-issue-123925.TLlyUi.rst @@ -0,0 +1,2 @@ +Fix building the :mod:`curses` module on platforms with libncurses but +without libncursesw. diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 040ffa81153ebe..92961af381b9cb 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -138,7 +138,7 @@ typedef chtype attr_t; /* No attr_t type is available */ #define STRICT_SYSV_CURSES #endif -#if NCURSES_EXT_FUNCS+0 >= 20170401 && NCURSES_EXT_COLORS+0 >= 20170401 +#if defined(HAVE_NCURSESW) && NCURSES_EXT_FUNCS+0 >= 20170401 && NCURSES_EXT_COLORS+0 >= 20170401 #define _NCURSES_EXTENDED_COLOR_FUNCS 1 #else #define _NCURSES_EXTENDED_COLOR_FUNCS 0 From a3711d1541c1b7987941b41d2247f87dae347117 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 2 Jan 2025 14:11:21 +0200 Subject: [PATCH 331/427] gh-124130: Fix a bug in matching regular expression \B in empty string (GH-127007) --- Doc/library/re.rst | 7 ++----- Doc/whatsnew/3.14.rst | 4 ++++ Lib/test/test_re.py | 13 +++++-------- .../2024-11-19-10-46-57.gh-issue-124130.OZ_vR5.rst | 4 ++++ Modules/_sre/sre_lib.h | 12 ------------ 5 files changed, 15 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-19-10-46-57.gh-issue-124130.OZ_vR5.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 9db6f1da3be4db..29387a429b844c 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -572,11 +572,8 @@ character ``'$'``. Word boundaries are determined by the current locale if the :py:const:`~re.LOCALE` flag is used. - .. note:: - - Note that ``\B`` does not match an empty string, which differs from - RE implementations in other programming languages such as Perl. - This behavior is kept for compatibility reasons. + .. versionchanged:: next + ``\B`` now matches empty input string. .. index:: single: \d; in regular expressions diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 4b3f1b2e8eed42..61f5ffdb6c89d1 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -245,6 +245,10 @@ Other language changes making it a :term:`generic type`. (Contributed by Brian Schubert in :gh:`126012`.) +* ``\B`` in :mod:`regular expression ` now matches empty input string. + Now it is always the opposite of ``\b``. + (Contributed by Serhiy Storchaka in :gh:`124130`.) + * iOS and macOS apps can now be configured to redirect ``stdout`` and ``stderr`` content to the system log. (Contributed by Russell Keith-Magee in :gh:`127592`.) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 0d3599be87f228..5538de60b2a03a 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -978,18 +978,15 @@ def test_word_boundaries(self): self.assertIsNone(re.fullmatch(br".+\B", b"abc", re.LOCALE)) self.assertIsNone(re.fullmatch(r".+\B", "ьюя")) self.assertTrue(re.fullmatch(r".+\B", "ьюя", re.ASCII)) - # However, an empty string contains no word boundaries, and also no - # non-boundaries. + # However, an empty string contains no word boundaries. self.assertIsNone(re.search(r"\b", "")) self.assertIsNone(re.search(r"\b", "", re.ASCII)) self.assertIsNone(re.search(br"\b", b"")) self.assertIsNone(re.search(br"\b", b"", re.LOCALE)) - # This one is questionable and different from the perlre behaviour, - # but describes current behavior. - self.assertIsNone(re.search(r"\B", "")) - self.assertIsNone(re.search(r"\B", "", re.ASCII)) - self.assertIsNone(re.search(br"\B", b"")) - self.assertIsNone(re.search(br"\B", b"", re.LOCALE)) + self.assertTrue(re.search(r"\B", "")) + self.assertTrue(re.search(r"\B", "", re.ASCII)) + self.assertTrue(re.search(br"\B", b"")) + self.assertTrue(re.search(br"\B", b"", re.LOCALE)) # A single word-character string has two boundaries, but no # non-boundary gaps. self.assertEqual(len(re.findall(r"\b", "a")), 2) diff --git a/Misc/NEWS.d/next/Library/2024-11-19-10-46-57.gh-issue-124130.OZ_vR5.rst b/Misc/NEWS.d/next/Library/2024-11-19-10-46-57.gh-issue-124130.OZ_vR5.rst new file mode 100644 index 00000000000000..a1d4fc8ff4c22f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-19-10-46-57.gh-issue-124130.OZ_vR5.rst @@ -0,0 +1,4 @@ +Fix a bug in matching regular expression ``\B`` in empty input string. +Now it is always the opposite of ``\b``. +To get an old behavior, use ``(?!\A\Z)\B``. +To get a new behavior in old Python versions, use ``(?!\b)``. diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index af4bfc56083bcb..df377905bfae0d 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -42,8 +42,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return ((void*) ptr == state->end); case SRE_AT_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -51,8 +49,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return thisp != thatp; case SRE_AT_NON_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -60,8 +56,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return thisp == thatp; case SRE_AT_LOC_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_LOC_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -69,8 +63,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return thisp != thatp; case SRE_AT_LOC_NON_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_LOC_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -78,8 +70,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return thisp == thatp; case SRE_AT_UNI_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_UNI_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -87,8 +77,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return thisp != thatp; case SRE_AT_UNI_NON_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_UNI_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? From 8e48a6edc75ca67a34924bbe54463ca913ae6e58 Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Thu, 2 Jan 2025 13:54:38 +0100 Subject: [PATCH 332/427] gh-126624: Expose error code ``XML_ERROR_NOT_STARTED`` of Expat >=2.6.4 (#126625) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose error code ``XML_ERROR_NOT_STARTED`` in `xml.parsers.expat.errors` which was introduced in Expat 2.6.4. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/pyexpat.rst | 7 +++++++ .../Library/2024-11-09-15-59-51.gh-issue-126624.bN53Va.rst | 2 ++ Modules/pyexpat.c | 5 ++++- 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-09-15-59-51.gh-issue-126624.bN53Va.rst diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index c0e9999f4b1270..0f3b58ef6ea5af 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -941,6 +941,13 @@ The ``errors`` module has the following attributes: has been breached. +.. data:: XML_ERROR_NOT_STARTED + + The parser was tried to be stopped or suspended before it started. + + .. versionadded:: next + + .. rubric:: Footnotes .. [1] The encoding string included in XML output should conform to the diff --git a/Misc/NEWS.d/next/Library/2024-11-09-15-59-51.gh-issue-126624.bN53Va.rst b/Misc/NEWS.d/next/Library/2024-11-09-15-59-51.gh-issue-126624.bN53Va.rst new file mode 100644 index 00000000000000..468840a651c253 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-09-15-59-51.gh-issue-126624.bN53Va.rst @@ -0,0 +1,2 @@ +Expose error code :data:`~xml.parsers.expat.errors.XML_ERROR_NOT_STARTED` +of Expat >=2.6.4 in :mod:`xml.parsers.expat.errors`. diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index cf7714e7656205..9931ca2a8d4749 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1767,7 +1767,10 @@ struct ErrorInfo error_info_of[] = { {"XML_ERROR_NO_BUFFER", "a successful prior call to function XML_GetBuffer is required"}, /* Added in 2.4.0. */ - {"XML_ERROR_AMPLIFICATION_LIMIT_BREACH", "limit on input amplification factor (from DTD and entities) breached"} + {"XML_ERROR_AMPLIFICATION_LIMIT_BREACH", "limit on input amplification factor (from DTD and entities) breached"}, + + /* Added in 2.6.4. */ + {"XML_ERROR_NOT_STARTED", "parser not started"}, }; static int From 9ba0528537b2befcd02a7fdc5904e0cfbc33a887 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 2 Jan 2025 13:00:26 +0000 Subject: [PATCH 333/427] gh-128404: Remove ``asyncio`` from ``test_builtin`` (#128403) Co-authored-by: Kumar Aditya --- Lib/test/test_builtin.py | 123 ++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 59 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index f98138391bc1a8..5f4eac5267622f 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1,7 +1,6 @@ # Python test set -- built-in functions import ast -import asyncio import builtins import collections import contextlib @@ -31,7 +30,8 @@ from types import AsyncGeneratorType, FunctionType, CellType from operator import neg from test import support -from test.support import (cpython_only, swap_attr, maybe_get_event_loop_policy) +from test.support import cpython_only, swap_attr +from test.support import async_yield, run_yielding_async_fn from test.support.import_helper import import_module from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok @@ -414,10 +414,6 @@ def test_compile_top_level_await_no_coro(self): msg=f"source={source} mode={mode}") - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "socket.accept is broken" - ) def test_compile_top_level_await(self): """Test whether code with top level await can be compiled. @@ -432,13 +428,25 @@ async def arange(n): for i in range(n): yield i + class Lock: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + pass + + async def sleep(delay, result=None): + assert delay == 0 + await async_yield(None) + return result + modes = ('single', 'exec') optimizations = (-1, 0, 1, 2) code_samples = [ - '''a = await asyncio.sleep(0, result=1)''', + '''a = await sleep(0, result=1)''', '''async for i in arange(1): a = 1''', - '''async with asyncio.Lock() as l: + '''async with Lock() as l: a = 1''', '''a = [x async for x in arange(2)][1]''', '''a = 1 in {x async for x in arange(2)}''', @@ -446,16 +454,16 @@ async def arange(n): '''a = [x async for x in arange(2) async for x in arange(2)][1]''', '''a = [x async for x in (x async for x in arange(5))][1]''', '''a, = [1 for x in {x async for x in arange(1)}]''', - '''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]''', + '''a = [await sleep(0, x) async for x in arange(2)][1]''', # gh-121637: Make sure we correctly handle the case where the # async code is optimized away - '''assert not await asyncio.sleep(0); a = 1''', + '''assert not await sleep(0); a = 1''', '''assert [x async for x in arange(1)]; a = 1''', '''assert {x async for x in arange(1)}; a = 1''', '''assert {x: x async for x in arange(1)}; a = 1''', ''' if (a := 1) and __debug__: - async with asyncio.Lock() as l: + async with Lock() as l: pass ''', ''' @@ -464,36 +472,31 @@ async def arange(n): pass ''', ] - policy = maybe_get_event_loop_policy() - try: - for mode, code_sample, optimize in product(modes, code_samples, optimizations): - with self.subTest(mode=mode, code_sample=code_sample, optimize=optimize): - source = dedent(code_sample) - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode, optimize=optimize) - - co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, - optimize=optimize) - - self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, - msg=f"source={source} mode={mode}") - - # test we can create and advance a function type - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - async_f = FunctionType(co, globals_) - asyncio.run(async_f()) - self.assertEqual(globals_['a'], 1) - - # test we can await-eval, - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - asyncio.run(eval(co, globals_)) - self.assertEqual(globals_['a'], 1) - finally: - asyncio._set_event_loop_policy(policy) + for mode, code_sample, optimize in product(modes, code_samples, optimizations): + with self.subTest(mode=mode, code_sample=code_sample, optimize=optimize): + source = dedent(code_sample) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + compile(source, '?', mode, optimize=optimize) + + co = compile(source, + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + optimize=optimize) + + self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, + msg=f"source={source} mode={mode}") + + # test we can create and advance a function type + globals_ = {'Lock': Lock, 'a': 0, 'arange': arange, 'sleep': sleep} + run_yielding_async_fn(FunctionType(co, globals_)) + self.assertEqual(globals_['a'], 1) + + # test we can await-eval, + globals_ = {'Lock': Lock, 'a': 0, 'arange': arange, 'sleep': sleep} + run_yielding_async_fn(lambda: eval(co, globals_)) + self.assertEqual(globals_['a'], 1) def test_compile_top_level_await_invalid_cases(self): # helper function just to check we can run top=level async-for @@ -501,6 +504,13 @@ async def arange(n): for i in range(n): yield i + class Lock: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + pass + modes = ('single', 'exec') code_samples = [ '''def f(): await arange(10)\n''', @@ -511,27 +521,22 @@ async def arange(n): a = 1 ''', '''def f(): - async with asyncio.Lock() as l: + async with Lock() as l: a = 1 ''' ] - policy = maybe_get_event_loop_policy() - try: - for mode, code_sample in product(modes, code_samples): - source = dedent(code_sample) - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) - - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) - finally: - asyncio._set_event_loop_policy(policy) + for mode, code_sample in product(modes, code_samples): + source = dedent(code_sample) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + compile(source, '?', mode) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + co = compile(source, + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) def test_compile_async_generator(self): """ @@ -542,7 +547,7 @@ def test_compile_async_generator(self): code = dedent("""async def ticker(): for i in range(10): yield i - await asyncio.sleep(0)""") + await sleep(0)""") co = compile(code, '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) glob = {} From c9d2bc6d7f6d74e0539afb0f7066997ae736dfc8 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:35:00 +0200 Subject: [PATCH 334/427] Revert "Doc: Show object descriptions in the table of contents (#125757)" (#128406) --- Doc/conf.py | 4 +--- Doc/tools/extensions/pyspecific.py | 1 - Doc/tools/static/sidebar-wrap.css | 6 ------ 3 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 Doc/tools/static/sidebar-wrap.css diff --git a/Doc/conf.py b/Doc/conf.py index 9cde394cbaed69..ae08c7fa288080 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -101,9 +101,7 @@ # Create table of contents entries for domain objects (e.g. functions, classes, # attributes, etc.). Default is True. -toc_object_entries = True -# Hide parents to tidy up long entries in sidebar -toc_object_entries_show_parents = 'hide' +toc_object_entries = False # Ignore any .rst files in the includes/ directory; # they're embedded in pages but not rendered as individual pages. diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index f4df7ec0839339..bcb8a421e32d09 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -434,6 +434,5 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) - app.add_css_file('sidebar-wrap.css') app.connect('env-check-consistency', patch_pairindextypes) return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/static/sidebar-wrap.css b/Doc/tools/static/sidebar-wrap.css deleted file mode 100644 index 0a80f516f28349..00000000000000 --- a/Doc/tools/static/sidebar-wrap.css +++ /dev/null @@ -1,6 +0,0 @@ -div.sphinxsidebarwrapper { - overflow-x: scroll; -} -div.sphinxsidebarwrapper li code { - overflow-wrap: normal; -} From 58e9f95c4aa970db32a94b9152b51ede22f823bd Mon Sep 17 00:00:00 2001 From: Zhikang Yan <2951256653@qq.com> Date: Thu, 2 Jan 2025 23:51:57 +0800 Subject: [PATCH 335/427] gh-128014: Fix passing default='' to the tkinter method wm_iconbitmap() (GH-128015) Co-authored-by: Serhiy Storchaka --- Lib/test/test_tkinter/test_misc.py | 28 ++++++++++++++++++- Lib/tkinter/__init__.py | 2 +- ...-12-18-00-07-50.gh-issue-128014.F3aUbz.rst | 2 ++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-18-00-07-50.gh-issue-128014.F3aUbz.rst diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 475edcbd5338a7..3362169391818b 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -4,7 +4,8 @@ from tkinter import TclError import enum from test import support -from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tk +from test.test_tkinter.support import (AbstractTkTest, AbstractDefaultRootTest, + requires_tk, get_tk_patchlevel) support.requires('gui') @@ -554,6 +555,31 @@ def test_wm_attribute(self): self.assertEqual(w.wm_attributes('alpha'), 1.0 if self.wantobjects else '1.0') + def test_wm_iconbitmap(self): + t = tkinter.Toplevel(self.root) + self.assertEqual(t.wm_iconbitmap(), '') + t.wm_iconbitmap('hourglass') + bug = False + if t._windowingsystem == 'aqua': + # Tk bug 13ac26b35dc55f7c37f70b39d59d7ef3e63017c8. + patchlevel = get_tk_patchlevel(t) + if patchlevel < (8, 6, 17) or (9, 0) <= patchlevel < (9, 0, 2): + bug = True + if not bug: + self.assertEqual(t.wm_iconbitmap(), 'hourglass') + self.assertEqual(self.root.wm_iconbitmap(), '') + t.wm_iconbitmap('') + self.assertEqual(t.wm_iconbitmap(), '') + + if t._windowingsystem == 'win32': + t.wm_iconbitmap(default='hourglass') + self.assertEqual(t.wm_iconbitmap(), 'hourglass') + self.assertEqual(self.root.wm_iconbitmap(), '') + t.wm_iconbitmap(default='') + self.assertEqual(t.wm_iconbitmap(), '') + + t.destroy() + class EventTest(AbstractTkTest, unittest.TestCase): diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index bfec04bb6c1e6e..d494c0c9687cd1 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2265,7 +2265,7 @@ def wm_iconbitmap(self, bitmap=None, default=None): explicitly. DEFAULT can be the relative path to a .ico file (example: root.iconbitmap(default='myicon.ico') ). See Tk documentation for more information.""" - if default: + if default is not None: return self.tk.call('wm', 'iconbitmap', self._w, '-default', default) else: return self.tk.call('wm', 'iconbitmap', self._w, bitmap) diff --git a/Misc/NEWS.d/next/Library/2024-12-18-00-07-50.gh-issue-128014.F3aUbz.rst b/Misc/NEWS.d/next/Library/2024-12-18-00-07-50.gh-issue-128014.F3aUbz.rst new file mode 100644 index 00000000000000..ef339a291f0ddd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-18-00-07-50.gh-issue-128014.F3aUbz.rst @@ -0,0 +1,2 @@ +Fix resetting the default window icon by passing ``default=''`` to the +:mod:`tkinter` method :meth:`!wm_iconbitmap`. From a626f9a67b76e5fe69677afd5f8317d8c61de8de Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Thu, 2 Jan 2025 08:55:33 -0800 Subject: [PATCH 336/427] Remove asserts that confuse `enum _framestate` with `enum _frameowner` (GH-124148) The `owner` field of `_PyInterpreterFrame` is supposed to be a member of `enum _frameowner`, but `FRAME_CLEARED` is a member of `enum _framestate`. At present, it happens that `FRAME_CLEARED` is not numerically equal to any member of `enum _frameowner`, but that could change in the future. The code that incorrectly assigned `owner = FRAME_CLEARED` was deleted in commit a53cc3f49463e50cb3e2b839b3a82e6bf7f73fee (GH-116687). Remove the incorrect checks for `owner != FRAME_CLEARED` as well. --- Python/frame.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/frame.c b/Python/frame.c index 9a865e57d97cc6..6eb32bcce0b799 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -40,7 +40,6 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame) // here. assert(frame->frame_obj == NULL); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); - assert(frame->owner != FRAME_CLEARED); f->f_frame = frame; frame->frame_obj = f; return f; @@ -51,7 +50,6 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) { assert(frame->owner != FRAME_OWNED_BY_CSTACK); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); - assert(frame->owner != FRAME_CLEARED); Py_ssize_t size = ((char*)frame->stackpointer) - (char *)frame; memcpy((_PyInterpreterFrame *)f->_f_frame_data, frame, size); frame = (_PyInterpreterFrame *)f->_f_frame_data; From c9356feef28e6dfc4dc32830d3427a5ae0e426e2 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 2 Jan 2025 13:56:01 -0500 Subject: [PATCH 337/427] gh-128400: Stop-the-world when manually calling `faulthandler` (GH-128422) --- Lib/test/test_faulthandler.py | 30 ++++++++++++++++++- ...-01-02-13-05-16.gh-issue-128400.5N43fF.rst | 2 ++ Modules/faulthandler.c | 5 ++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-02-13-05-16.gh-issue-128400.5N43fF.rst diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 60815be96e14eb..fd56dee5d842ac 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -7,7 +7,7 @@ import subprocess import sys from test import support -from test.support import os_helper, script_helper, is_android, MS_WINDOWS +from test.support import os_helper, script_helper, is_android, MS_WINDOWS, threading_helper import tempfile import unittest from textwrap import dedent @@ -896,6 +896,34 @@ def test_cancel_later_without_dump_traceback_later(self): self.assertEqual(output, []) self.assertEqual(exitcode, 0) + @threading_helper.requires_working_threading() + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful if the GIL is disabled") + def test_free_threaded_dump_traceback(self): + # gh-128400: Other threads need to be paused to invoke faulthandler + code = dedent(""" + import faulthandler + from threading import Thread, Event + + class Waiter(Thread): + def __init__(self): + Thread.__init__(self) + self.running = Event() + self.stop = Event() + + def run(self): + self.running.set() + self.stop.wait() + + for _ in range(100): + waiter = Waiter() + waiter.start() + waiter.running.wait() + faulthandler.dump_traceback(all_threads=True) + waiter.stop.set() + waiter.join() + """) + _, exitcode = self.get_output(code) + self.assertEqual(exitcode, 0) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-01-02-13-05-16.gh-issue-128400.5N43fF.rst b/Misc/NEWS.d/next/Library/2025-01-02-13-05-16.gh-issue-128400.5N43fF.rst new file mode 100644 index 00000000000000..4033dea4eaf7bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-02-13-05-16.gh-issue-128400.5N43fF.rst @@ -0,0 +1,2 @@ +Fix crash when using :func:`faulthandler.dump_traceback` while other threads +are active on the :term:`free threaded ` build. diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index b62362f277797e..2d16028a5232d0 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -237,7 +237,12 @@ faulthandler_dump_traceback_py(PyObject *self, return NULL; if (all_threads) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + /* gh-128400: Accessing other thread states while they're running + * isn't safe if those threads are running. */ + _PyEval_StopTheWorld(interp); errmsg = _Py_DumpTracebackThreads(fd, NULL, tstate); + _PyEval_StartTheWorld(interp); if (errmsg != NULL) { PyErr_SetString(PyExc_RuntimeError, errmsg); return NULL; From 8eebe4e6d02bb4ad3f1ca6c52624186903dce893 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 2 Jan 2025 14:02:54 -0500 Subject: [PATCH 338/427] gh-128212: Fix race in `_PyUnicode_CheckConsistency` (GH-128367) There was a data race on the utf8 field between `PyUnicode_SET_UTF8` and `_PyUnicode_CheckConsistency`. Use the `_PyUnicode_UTF8()` accessor, which uses an atomic load internally, to avoid the data race. --- Objects/unicodeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1aab9cf37768a8..9f0a4d4785eda6 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -688,7 +688,7 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) || kind == PyUnicode_2BYTE_KIND || kind == PyUnicode_4BYTE_KIND); CHECK(ascii->state.ascii == 0); - CHECK(compact->utf8 != data); + CHECK(_PyUnicode_UTF8(op) != data); } else { PyUnicodeObject *unicode = _PyUnicodeObject_CAST(op); From e7adeecc2b318505eb53bc779320f028be40cccc Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 3 Jan 2025 00:40:24 -0700 Subject: [PATCH 339/427] gh-128426: Mention PySequence_Fast in free-threading C API HOWTO (#128428) --- Doc/howto/free-threading-extensions.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index c1ad42e7e55ee5..95f214179bfb0e 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -96,8 +96,10 @@ Most of the C API is thread-safe, but there are some exceptions. * **Struct Fields**: Accessing fields in Python C API objects or structs directly is not thread-safe if the field may be concurrently modified. -* **Macros**: Accessor macros like :c:macro:`PyList_GET_ITEM` and - :c:macro:`PyList_SET_ITEM` do not perform any error checking or locking. +* **Macros**: Accessor macros like :c:macro:`PyList_GET_ITEM`, + :c:macro:`PyList_SET_ITEM`, and macros like + :c:macro:`PySequence_Fast_GET_SIZE` that use the object returned by + :c:func:`PySequence_Fast` do not perform any error checking or locking. These macros are not thread-safe if the container object may be modified concurrently. * **Borrowed References**: C API functions that return From b49c68a1b3dfd2c2567c38b2d044c4a1c14a26a7 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 3 Jan 2025 07:50:24 +0000 Subject: [PATCH 340/427] gh-128404: remove asyncio from test_type_params (#128436) --- Lib/test/test_type_params.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 89f836cf722966..0f393def827271 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1,11 +1,10 @@ import annotationlib -import asyncio import textwrap import types import unittest import pickle import weakref -from test.support import requires_working_socket, check_syntax_error, run_code +from test.support import check_syntax_error, run_code, run_no_yield_async_fn from typing import Generic, NoDefault, Sequence, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, get_args @@ -1051,7 +1050,6 @@ def generator2[B](): self.assertIsInstance(c, TypeVar) self.assertEqual(c.__name__, "C") - @requires_working_socket() def test_typevar_coroutine(self): def get_coroutine[A](): async def coroutine[B](): @@ -1060,8 +1058,7 @@ async def coroutine[B](): co = get_coroutine() - self.addCleanup(asyncio._set_event_loop_policy, None) - a, b = asyncio.run(co()) + a, b = run_no_yield_async_fn(co) self.assertIsInstance(a, TypeVar) self.assertEqual(a.__name__, "A") From 0706bab1c0985761cdbc07ab448f98c717276b36 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Fri, 3 Jan 2025 09:20:56 +0100 Subject: [PATCH 341/427] gh-128133: use relaxed atomics for hash of bytes (#128412) --- Objects/bytesobject.c | 70 ++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 533089d25cd73a..90e8a9af88b4a3 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -51,6 +51,33 @@ static inline PyObject* bytes_get_empty(void) } +static inline void +set_ob_shash(PyBytesObject *a, Py_hash_t hash) +{ +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ssize_relaxed(&a->ob_shash, hash); +#else + a->ob_shash = hash; +#endif +_Py_COMP_DIAG_POP +} + +static inline Py_hash_t +get_ob_shash(PyBytesObject *a) +{ +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&a->ob_shash); +#else + return a->ob_shash; +#endif +_Py_COMP_DIAG_POP +} + + /* For PyBytes_FromString(), the parameter 'str' points to a null-terminated string containing exactly 'size' bytes. @@ -98,10 +125,7 @@ _PyBytes_FromSize(Py_ssize_t size, int use_calloc) return PyErr_NoMemory(); } _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - op->ob_shash = -1; -_Py_COMP_DIAG_POP + set_ob_shash(op, -1); if (!use_calloc) { op->ob_sval[size] = '\0'; } @@ -165,10 +189,7 @@ PyBytes_FromString(const char *str) return PyErr_NoMemory(); } _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - op->ob_shash = -1; -_Py_COMP_DIAG_POP + set_ob_shash(op, -1); memcpy(op->ob_sval, str, size+1); return (PyObject *) op; } @@ -1485,10 +1506,7 @@ bytes_repeat(PyObject *self, Py_ssize_t n) return PyErr_NoMemory(); } _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - op->ob_shash = -1; -_Py_COMP_DIAG_POP + set_ob_shash(op, -1); op->ob_sval[size] = '\0'; _PyBytes_Repeat(op->ob_sval, size, a->ob_sval, Py_SIZE(a)); @@ -1597,14 +1615,13 @@ static Py_hash_t bytes_hash(PyObject *self) { PyBytesObject *a = _PyBytes_CAST(self); -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - if (a->ob_shash == -1) { + Py_hash_t hash = get_ob_shash(a); + if (hash == -1) { /* Can't fail */ - a->ob_shash = Py_HashBuffer(a->ob_sval, Py_SIZE(a)); + hash = Py_HashBuffer(a->ob_sval, Py_SIZE(a)); + set_ob_shash(a, hash); } - return a->ob_shash; -_Py_COMP_DIAG_POP + return hash; } static PyObject* @@ -3004,10 +3021,7 @@ bytes_alloc(PyTypeObject *self, Py_ssize_t nitems) if (obj == NULL) { return NULL; } -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - obj->ob_shash = -1; -_Py_COMP_DIAG_POP + set_ob_shash(obj, -1); return (PyObject*)obj; } @@ -3024,11 +3038,8 @@ bytes_subtype_new(PyTypeObject *type, PyObject *tmp) if (pnew != NULL) { memcpy(PyBytes_AS_STRING(pnew), PyBytes_AS_STRING(tmp), n+1); -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - ((PyBytesObject *)pnew)->ob_shash = - ((PyBytesObject *)tmp)->ob_shash; -_Py_COMP_DIAG_POP + set_ob_shash((PyBytesObject *)pnew, + get_ob_shash((PyBytesObject *)tmp)); } return pnew; } @@ -3221,10 +3232,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) sv = (PyBytesObject *) *pv; Py_SET_SIZE(sv, newsize); sv->ob_sval[newsize] = '\0'; -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - sv->ob_shash = -1; /* invalidate cached hash value */ -_Py_COMP_DIAG_POP + set_ob_shash(sv, -1); /* invalidate cached hash value */ return 0; } From bb73426cafb78691b249ffa50f1872ab6f899d4a Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 3 Jan 2025 09:57:15 +0000 Subject: [PATCH 342/427] gh-108202: Document calendar.TextCalendar formatting helpers (#127608) Document the following TextCalendar methods: - formatday() - formatmonthname() - formatweekday() - formatweekheader() --- Doc/library/calendar.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 9a4395ecb5084d..97ca34b6c6184c 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -138,6 +138,14 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :class:`TextCalendar` instances have the following methods: + + .. method:: formatday(theday, weekday, width) + + Return a string representing a single day formatted with the given *width*. + If *theday* is ``0``, return a string of spaces of + the specified width, representing an empty day. The *weekday* parameter + is unused. + .. method:: formatweek(theweek, w=0) Return a single week in a string with no newline. If *w* is provided, it @@ -145,6 +153,17 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is on the first weekday as specified in the constructor or set by the :meth:`setfirstweekday` method. + .. method:: formatweekday(weekday, width) + + Return a string representing the name of a single weekday formatted to + the specified *width*. The *weekday* parameter is an integer representing + the day of the week, where ``0`` is Monday and ``6`` is Sunday. + + .. method:: formatweekheader(width) + + Return a string containing the header row of weekday names, formatted + with the given *width* for each column. The names depend on the locale + settings and are padded to the specified width. .. method:: formatmonth(theyear, themonth, w=0, l=0) @@ -154,6 +173,12 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is on the first weekday as specified in the constructor or set by the :meth:`setfirstweekday` method. + .. method:: formatmonthname(theyear, themonth, width=0, withyear=True) + + Return a string representing the month's name centered within the + specified *width*. If *withyear* is ``True``, include the year in the + output. The *theyear* and *themonth* parameters specify the year + and month for the name to be formatted respectively. .. method:: prmonth(theyear, themonth, w=0, l=0) @@ -445,7 +470,7 @@ The :mod:`calendar` module exports the following data attributes: A sequence that represents the months of the year in the current locale. This follows normal convention of January being month number 1, so it has a length of - 13 and ``month_name[0]`` is the empty string. + 13 and ``month_name[0]`` is the empty string. >>> import calendar >>> list(calendar.month_name) From bb2dfadb9221fa3035fda42a2c153c831013e3d3 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Fri, 3 Jan 2025 04:04:03 -0600 Subject: [PATCH 343/427] gh-128104: Remove `Py_STRFTIME_C99_SUPPORT`; require C99-compliant strftime (#128106) --- .../2024-12-20-09-03-22.gh-issue-128104.m_SoVx.rst | 3 +++ Modules/_datetimemodule.c | 6 ------ configure | 14 ++++---------- configure.ac | 11 +++-------- pyconfig.h.in | 3 --- 5 files changed, 10 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-12-20-09-03-22.gh-issue-128104.m_SoVx.rst diff --git a/Misc/NEWS.d/next/Build/2024-12-20-09-03-22.gh-issue-128104.m_SoVx.rst b/Misc/NEWS.d/next/Build/2024-12-20-09-03-22.gh-issue-128104.m_SoVx.rst new file mode 100644 index 00000000000000..c3a47fbecd1dad --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-20-09-03-22.gh-issue-128104.m_SoVx.rst @@ -0,0 +1,3 @@ +Remove ``Py_STRFTIME_C99_SUPPORT`` conditions in favor of requiring C99 +:manpage:`strftime(3)` specifier support at build time. When cross-compiling, +there is no build time check and support is assumed. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index b1102984cb5e9e..368d10411366c4 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1912,9 +1912,7 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, } #ifdef Py_NORMALIZE_CENTURY else if (ch == 'Y' || ch == 'G' -#ifdef Py_STRFTIME_C99_SUPPORT || ch == 'F' || ch == 'C' -#endif ) { /* 0-pad year with century as necessary */ PyObject *item = PySequence_GetItem(timetuple, 0); @@ -1952,15 +1950,11 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, * +6 to accommodate dashes, 2-digit month and day for %F. */ char buf[SIZEOF_LONG * 5 / 2 + 2 + 6]; Py_ssize_t n = PyOS_snprintf(buf, sizeof(buf), -#ifdef Py_STRFTIME_C99_SUPPORT ch == 'F' ? "%04ld-%%m-%%d" : -#endif "%04ld", year_long); -#ifdef Py_STRFTIME_C99_SUPPORT if (ch == 'C') { n -= 2; } -#endif if (_PyUnicodeWriter_WriteSubstring(&writer, format, start, end) < 0) { goto Error; } diff --git a/configure b/configure index 3d2c60213db591..2c3046e89bcb65 100755 --- a/configure +++ b/configure @@ -26436,8 +26436,8 @@ printf "%s\n" "#define Py_NORMALIZE_CENTURY 1" >>confdefs.h fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C99-specific strftime specifiers are supported" >&5 -printf %s "checking whether C99-specific strftime specifiers are supported... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C99-compatible strftime specifiers are supported" >&5 +printf %s "checking whether C99-compatible strftime specifiers are supported... " >&6; } if test ${ac_cv_strftime_c99_support+y} then : printf %s "(cached) " >&6 @@ -26445,7 +26445,7 @@ else $as_nop if test "$cross_compiling" = yes then : - ac_cv_strftime_c99_support=no + ac_cv_strftime_c99_support= else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -26472,7 +26472,7 @@ if ac_fn_c_try_run "$LINENO" then : ac_cv_strftime_c99_support=yes else $as_nop - ac_cv_strftime_c99_support=no + as_fn_error $? "Python requires C99-compatible strftime specifiers" "$LINENO" 5 fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext @@ -26481,12 +26481,6 @@ fi fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_strftime_c99_support" >&5 printf "%s\n" "$ac_cv_strftime_c99_support" >&6; } -if test "$ac_cv_strftime_c99_support" = yes -then - -printf "%s\n" "#define Py_STRFTIME_C99_SUPPORT 1" >>confdefs.h - -fi have_curses=no have_panel=no diff --git a/configure.ac b/configure.ac index ee034e5a9621df..50b130f2c802b5 100644 --- a/configure.ac +++ b/configure.ac @@ -6672,7 +6672,7 @@ then [Define if year with century should be normalized for strftime.]) fi -AC_CACHE_CHECK([whether C99-specific strftime specifiers are supported], [ac_cv_strftime_c99_support], [ +AC_CACHE_CHECK([whether C99-compatible strftime specifiers are supported], [ac_cv_strftime_c99_support], [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include @@ -6692,13 +6692,8 @@ int main(void) } ]])], [ac_cv_strftime_c99_support=yes], -[ac_cv_strftime_c99_support=no], -[ac_cv_strftime_c99_support=no])]) -if test "$ac_cv_strftime_c99_support" = yes -then - AC_DEFINE([Py_STRFTIME_C99_SUPPORT], [1], - [Define if C99-specific strftime specifiers are supported.]) -fi +[AC_MSG_ERROR([Python requires C99-compatible strftime specifiers])], +[ac_cv_strftime_c99_support=])]) dnl check for ncursesw/ncurses and panelw/panel dnl NOTE: old curses is not detected. diff --git a/pyconfig.h.in b/pyconfig.h.in index 1ca83fd2f2ca1b..ca08f087a85e9f 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1715,9 +1715,6 @@ /* Define if you want to enable internal statistics gathering. */ #undef Py_STATS -/* Define if C99-specific strftime specifiers are supported. */ -#undef Py_STRFTIME_C99_SUPPORT - /* The version of SunOS/Solaris as reported by `uname -r' without the dot. */ #undef Py_SUNOS_VERSION From 830e10651b1f45cd0af36ff611397b9f53171220 Mon Sep 17 00:00:00 2001 From: jb2170 Date: Fri, 3 Jan 2025 10:32:36 +0000 Subject: [PATCH 344/427] gh-127529: Correct asyncio's `accept_connection` behaviour for handling `ConnectionAbortedError` (#127532) Co-authored-by: Kumar Aditya --- Lib/asyncio/selector_events.py | 10 +++++--- Lib/test/test_asyncio/test_selector_events.py | 25 +++++++++++++++++++ ...-12-02-19-13-19.gh-issue-127529.Pj1Xtf.rst | 4 +++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-02-19-13-19.gh-issue-127529.Pj1Xtf.rst diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index f1ab9b12d69a5d..50992a607b3a1c 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -180,9 +180,13 @@ def _accept_connection( logger.debug("%r got a new connection from %r: %r", server, addr, conn) conn.setblocking(False) - except (BlockingIOError, InterruptedError, ConnectionAbortedError): - # Early exit because the socket accept buffer is empty. - return None + except ConnectionAbortedError: + # Discard connections that were aborted before accept(). + continue + except (BlockingIOError, InterruptedError): + # Early exit because of a signal or + # the socket accept buffer is empty. + return except OSError as exc: # There's nowhere to send the error, so just log it. if exc.errno in (errno.EMFILE, errno.ENFILE, diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index f984dc96415ba3..c9217d04bcd322 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -364,6 +364,31 @@ def test_accept_connection_multiple(self): self.loop.run_until_complete(asyncio.sleep(0)) self.assertEqual(sock.accept.call_count, backlog) + def test_accept_connection_skip_connectionabortederror(self): + sock = mock.Mock() + + def mock_sock_accept(): + # mock accept(2) returning -ECONNABORTED every-other + # time that it's called. This applies most to OpenBSD + # whose sockets generate this errno more reproducibly than + # Linux and other OS. + if sock.accept.call_count % 2 == 0: + raise ConnectionAbortedError + return (mock.Mock(), mock.Mock()) + + sock.accept.side_effect = mock_sock_accept + backlog = 100 + # test that _accept_connection's loop calls sock.accept + # all 100 times, continuing past ConnectionAbortedError + # instead of unnecessarily returning early + mock_obj = mock.patch.object + with mock_obj(self.loop, '_accept_connection2') as accept2_mock: + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + # as in test_accept_connection_multiple avoid task pending + # warnings by using asyncio.sleep(0) + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(sock.accept.call_count, backlog) class SelectorTransportTests(test_utils.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-12-02-19-13-19.gh-issue-127529.Pj1Xtf.rst b/Misc/NEWS.d/next/Library/2024-12-02-19-13-19.gh-issue-127529.Pj1Xtf.rst new file mode 100644 index 00000000000000..26f2fd5923ab7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-02-19-13-19.gh-issue-127529.Pj1Xtf.rst @@ -0,0 +1,4 @@ +Correct behavior of +:func:`!asyncio.selector_events.BaseSelectorEventLoop._accept_connection` +in handling :exc:`ConnectionAbortedError` in a loop. This improves +performance on OpenBSD. From 4ed36d6efb3e3bc613045016dee594d671997709 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 3 Jan 2025 10:33:35 +0000 Subject: [PATCH 345/427] gh-128404: Remove `asyncio` from `Lib/test/test_sys_settrace` (#128435) --- Lib/test/test_sys_settrace.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index e5cf88177f7131..28c2c681babe18 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -6,8 +6,7 @@ import difflib import gc from functools import wraps -import asyncio -from test.support import import_helper, requires_subprocess +from test.support import import_helper, requires_subprocess, run_no_yield_async_fn import contextlib import os import tempfile @@ -19,8 +18,6 @@ except ImportError: _testinternalcapi = None -support.requires_working_socket(module=True) - class tracecontext: """Context manager that traces its enter and exit.""" def __init__(self, output, value): @@ -2067,10 +2064,9 @@ def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, stack.enter_context(self.assertRaisesRegex(*error)) if warning is not None: stack.enter_context(self.assertWarnsRegex(*warning)) - asyncio.run(func(output)) + run_no_yield_async_fn(func, output) sys.settrace(None) - asyncio._set_event_loop_policy(None) self.compare_jump_output(expected, output) def jump_test(jumpFrom, jumpTo, expected, error=None, event='line', warning=None): From 8abd6cef68a0582a4d912be76caddd9da5d55ccd Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 3 Jan 2025 12:37:54 +0100 Subject: [PATCH 346/427] gh-115765: Upgrade to GNU Autoconf 2.72 (#128411) --- .github/workflows/build.yml | 4 +- Doc/whatsnew/3.14.rst | 3 + ...-01-02-12-50-46.gh-issue-115765.jko7Fg.rst | 2 + Tools/build/regen-configure.sh | 2 +- config.guess | 107 +- configure | 6885 ++++++++++------- install-sh | 8 +- pyconfig.h.in | 606 +- 8 files changed, 4511 insertions(+), 3106 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-01-02-12-50-46.gh-issue-115765.jko7Fg.rst diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8787402ccc4423..9adf860632e8a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: # reproducible: to get the same tools versions (autoconf, aclocal, ...) runs-on: ubuntu-24.04 container: - image: ghcr.io/python/autoconf:2024.11.11.11786316759 + image: ghcr.io/python/autoconf:2025.01.02.12581854023 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' @@ -63,7 +63,7 @@ jobs: run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Check Autoconf and aclocal versions run: | - grep "Generated by GNU Autoconf 2.71" configure + grep "Generated by GNU Autoconf 2.72" configure grep "aclocal 1.16.5" aclocal.m4 grep -q "runstatedir" configure grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 61f5ffdb6c89d1..69c356ba60e6d4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1069,6 +1069,9 @@ Changes in the Python API Build changes ============= +* GNU Autoconf 2.72 is now required to generate :file:`!configure`. + (Contributed by Erlend Aasland in :gh:`115765`.) + PEP 761: Discontinuation of PGP signatures ------------------------------------------ diff --git a/Misc/NEWS.d/next/Build/2025-01-02-12-50-46.gh-issue-115765.jko7Fg.rst b/Misc/NEWS.d/next/Build/2025-01-02-12-50-46.gh-issue-115765.jko7Fg.rst new file mode 100644 index 00000000000000..34618c2c1288bc --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-01-02-12-50-46.gh-issue-115765.jko7Fg.rst @@ -0,0 +1,2 @@ +GNU Autoconf 2.72 is now required to generate :file:`!configure`. +Patch by Erlend Aasland. diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh index d2a613b1e40dc1..c7683eb36763af 100755 --- a/Tools/build/regen-configure.sh +++ b/Tools/build/regen-configure.sh @@ -5,7 +5,7 @@ set -e -x # The check_autoconf_regen job of .github/workflows/build.yml must kept in # sync with this script. Use the same container image than the job so the job # doesn't need to run autoreconf in a container. -IMAGE="ghcr.io/python/autoconf:2024.11.11.11786316759" +IMAGE="ghcr.io/python/autoconf:2025.01.02.12581854023" AUTORECONF="autoreconf -ivf -Werror" WORK_DIR="/src" diff --git a/config.guess b/config.guess index e81d3ae7c210ba..cdfc4392047ce3 100755 --- a/config.guess +++ b/config.guess @@ -1,14 +1,14 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2021 Free Software Foundation, Inc. +# Copyright 1992-2023 Free Software Foundation, Inc. # shellcheck disable=SC2006,SC2268 # see below for rationale -timestamp='2021-06-03' +timestamp='2023-08-22' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or +# the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -47,7 +47,7 @@ me=`echo "$0" | sed -e 's,.*/,,'` usage="\ Usage: $0 [OPTION] -Output the configuration name of the system \`$me' is run on. +Output the configuration name of the system '$me' is run on. Options: -h, --help print this help, then exit @@ -60,13 +60,13 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2021 Free Software Foundation, Inc. +Copyright 1992-2023 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -102,8 +102,8 @@ GUESS= # temporary files to be created and, as you can see below, it is a # headache to deal with in a portable fashion. -# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still -# use `HOST_CC' if defined, but it is deprecated. +# Historically, 'CC_FOR_BUILD' used to be named 'HOST_CC'. We still +# use 'HOST_CC' if defined, but it is deprecated. # Portable tmp directory creation inspired by the Autoconf team. @@ -155,6 +155,9 @@ Linux|GNU|GNU/*) set_cc_for_build cat <<-EOF > "$dummy.c" + #if defined(__ANDROID__) + LIBC=android + #else #include #if defined(__UCLIBC__) LIBC=uclibc @@ -169,6 +172,7 @@ Linux|GNU|GNU/*) LIBC=musl #endif #endif + #endif EOF cc_set_libc=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` eval "$cc_set_libc" @@ -437,7 +441,7 @@ case $UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION in # This test works for both compilers. if test "$CC_FOR_BUILD" != no_compiler_found; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -m64 -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then SUN_ARCH=x86_64 @@ -459,7 +463,7 @@ case $UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION in UNAME_RELEASE=`uname -v` ;; esac - # Japanese Language versions have a version number like `4.1.3-JL'. + # Japanese Language versions have a version number like '4.1.3-JL'. SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/-/_/'` GUESS=sparc-sun-sunos$SUN_REL ;; @@ -904,7 +908,7 @@ EOF fi ;; *:FreeBSD:*:*) - UNAME_PROCESSOR=`/usr/bin/uname -p` + UNAME_PROCESSOR=`uname -p` case $UNAME_PROCESSOR in amd64) UNAME_PROCESSOR=x86_64 ;; @@ -929,6 +933,9 @@ EOF i*:PW*:*) GUESS=$UNAME_MACHINE-pc-pw32 ;; + *:SerenityOS:*:*) + GUESS=$UNAME_MACHINE-pc-serenity + ;; *:Interix*:*) case $UNAME_MACHINE in x86) @@ -963,11 +970,37 @@ EOF GNU_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` GUESS=$UNAME_MACHINE-unknown-$GNU_SYS$GNU_REL-$LIBC ;; + x86_64:[Mm]anagarm:*:*|i?86:[Mm]anagarm:*:*) + GUESS="$UNAME_MACHINE-pc-managarm-mlibc" + ;; + *:[Mm]anagarm:*:*) + GUESS="$UNAME_MACHINE-unknown-managarm-mlibc" + ;; *:Minix:*:*) GUESS=$UNAME_MACHINE-unknown-minix ;; aarch64:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + set_cc_for_build + CPU=$UNAME_MACHINE + LIBCABI=$LIBC + if test "$CC_FOR_BUILD" != no_compiler_found; then + ABI=64 + sed 's/^ //' << EOF > "$dummy.c" + #ifdef __ARM_EABI__ + #ifdef __ARM_PCS_VFP + ABI=eabihf + #else + ABI=eabi + #endif + #endif +EOF + cc_set_abi=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^ABI' | sed 's, ,,g'` + eval "$cc_set_abi" + case $ABI in + eabi | eabihf) CPU=armv8l; LIBCABI=$LIBC$ABI ;; + esac + fi + GUESS=$CPU-unknown-linux-$LIBCABI ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be @@ -1033,7 +1066,16 @@ EOF k1om:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC ;; - loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*) + kvx:Linux:*:*) + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; + kvx:cos:*:*) + GUESS=$UNAME_MACHINE-unknown-cos + ;; + kvx:mbr:*:*) + GUESS=$UNAME_MACHINE-unknown-mbr + ;; + loongarch32:Linux:*:* | loongarch64:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC ;; m32r*:Linux:*:*) @@ -1148,16 +1190,27 @@ EOF ;; x86_64:Linux:*:*) set_cc_for_build + CPU=$UNAME_MACHINE LIBCABI=$LIBC if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_X32 >/dev/null - then - LIBCABI=${LIBC}x32 - fi + ABI=64 + sed 's/^ //' << EOF > "$dummy.c" + #ifdef __i386__ + ABI=x86 + #else + #ifdef __ILP32__ + ABI=x32 + #endif + #endif +EOF + cc_set_abi=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^ABI' | sed 's, ,,g'` + eval "$cc_set_abi" + case $ABI in + x86) CPU=i686 ;; + x32) LIBCABI=${LIBC}x32 ;; + esac fi - GUESS=$UNAME_MACHINE-pc-linux-$LIBCABI + GUESS=$CPU-pc-linux-$LIBCABI ;; xtensa*:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC @@ -1177,7 +1230,7 @@ EOF GUESS=$UNAME_MACHINE-pc-sysv4.2uw$UNAME_VERSION ;; i*86:OS/2:*:*) - # If we were able to find `uname', then EMX Unix compatibility + # If we were able to find 'uname', then EMX Unix compatibility # is probably installed. GUESS=$UNAME_MACHINE-pc-os2-emx ;; @@ -1318,7 +1371,7 @@ EOF GUESS=ns32k-sni-sysv fi ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + PENTIUM:*:4.0*:*) # Unisys 'ClearPath HMP IX 4000' SVR4/MP effort # says GUESS=i586-unisys-sysv4 ;; @@ -1364,8 +1417,11 @@ EOF BePC:Haiku:*:*) # Haiku running on Intel PC compatible. GUESS=i586-pc-haiku ;; - x86_64:Haiku:*:*) - GUESS=x86_64-unknown-haiku + ppc:Haiku:*:*) # Haiku running on Apple PowerPC + GUESS=powerpc-apple-haiku + ;; + *:Haiku:*:*) # Haiku modern gcc (not bound by BeOS compat) + GUESS=$UNAME_MACHINE-unknown-haiku ;; SX-4:SUPER-UX:*:*) GUESS=sx4-nec-superux$UNAME_RELEASE @@ -1522,6 +1578,9 @@ EOF i*86:rdos:*:*) GUESS=$UNAME_MACHINE-pc-rdos ;; + i*86:Fiwix:*:*) + GUESS=$UNAME_MACHINE-pc-fiwix + ;; *:AROS:*:*) GUESS=$UNAME_MACHINE-unknown-aros ;; diff --git a/configure b/configure index 2c3046e89bcb65..86b96d6208fe40 100755 --- a/configure +++ b/configure @@ -1,11 +1,11 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for python 3.14. +# Generated by GNU Autoconf 2.72 for python 3.14. # # Report bugs to . # # -# Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation, +# Copyright (C) 1992-1996, 1998-2017, 2020-2023 Free Software Foundation, # Inc. # # @@ -17,7 +17,6 @@ # Be more Bourne compatible DUALCASE=1; export DUALCASE # for MKS sh -as_nop=: if test ${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1 then : emulate sh @@ -26,12 +25,13 @@ then : # is contrary to our usage. Disable this feature. alias -g '${1+"$@"}'='"$@"' setopt NO_GLOB_SUBST -else $as_nop - case `(set -o) 2>/dev/null` in #( +else case e in #( + e) case `(set -o) 2>/dev/null` in #( *posix*) : set -o posix ;; #( *) : ;; +esac ;; esac fi @@ -103,7 +103,7 @@ IFS=$as_save_IFS ;; esac -# We did not find ourselves, most probably we were run as `sh COMMAND' +# We did not find ourselves, most probably we were run as 'sh COMMAND' # in which case we are not to be found in the path. if test "x$as_myself" = x; then as_myself=$0 @@ -133,15 +133,14 @@ case $- in # (((( esac exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} # Admittedly, this is quite paranoid, since all the known shells bail -# out after a failed `exec'. +# out after a failed 'exec'. printf "%s\n" "$0: could not re-execute with $CONFIG_SHELL" >&2 exit 255 fi # We don't want this to propagate to other subprocesses. { _as_can_reexec=; unset _as_can_reexec;} if test "x$CONFIG_SHELL" = x; then - as_bourne_compatible="as_nop=: -if test \${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1 + as_bourne_compatible="if test \${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1 then : emulate sh NULLCMD=: @@ -149,12 +148,13 @@ then : # is contrary to our usage. Disable this feature. alias -g '\${1+\"\$@\"}'='\"\$@\"' setopt NO_GLOB_SUBST -else \$as_nop - case \`(set -o) 2>/dev/null\` in #( +else case e in #( + e) case \`(set -o) 2>/dev/null\` in #( *posix*) : set -o posix ;; #( *) : ;; +esac ;; esac fi " @@ -172,8 +172,9 @@ as_fn_ret_failure && { exitcode=1; echo as_fn_ret_failure succeeded.; } if ( set x; as_fn_ret_success y && test x = \"\$1\" ) then : -else \$as_nop - exitcode=1; echo positional parameters were not saved. +else case e in #( + e) exitcode=1; echo positional parameters were not saved. ;; +esac fi test x\$exitcode = x0 || exit 1 blah=\$(echo \$(echo blah)) @@ -187,14 +188,15 @@ test \$(( 1 + 1 )) = 2 || exit 1" if (eval "$as_required") 2>/dev/null then : as_have_required=yes -else $as_nop - as_have_required=no +else case e in #( + e) as_have_required=no ;; +esac fi if test x$as_have_required = xyes && (eval "$as_suggested") 2>/dev/null then : -else $as_nop - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +else case e in #( + e) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR as_found=false for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH do @@ -227,12 +229,13 @@ IFS=$as_save_IFS if $as_found then : -else $as_nop - if { test -f "$SHELL" || test -f "$SHELL.exe"; } && +else case e in #( + e) if { test -f "$SHELL" || test -f "$SHELL.exe"; } && as_run=a "$SHELL" -c "$as_bourne_compatible""$as_required" 2>/dev/null then : CONFIG_SHELL=$SHELL as_have_required=yes -fi +fi ;; +esac fi @@ -254,7 +257,7 @@ case $- in # (((( esac exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} # Admittedly, this is quite paranoid, since all the known shells bail -# out after a failed `exec'. +# out after a failed 'exec'. printf "%s\n" "$0: could not re-execute with $CONFIG_SHELL" >&2 exit 255 fi @@ -274,7 +277,8 @@ $0: message. Then install a modern shell, or manually run $0: the script under such a shell if you do have one." fi exit 1 -fi +fi ;; +esac fi fi SHELL=${CONFIG_SHELL-/bin/sh} @@ -313,14 +317,6 @@ as_fn_exit () as_fn_set_status $1 exit $1 } # as_fn_exit -# as_fn_nop -# --------- -# Do nothing but, unlike ":", preserve the value of $?. -as_fn_nop () -{ - return $? -} -as_nop=as_fn_nop # as_fn_mkdir_p # ------------- @@ -389,11 +385,12 @@ then : { eval $1+=\$2 }' -else $as_nop - as_fn_append () +else case e in #( + e) as_fn_append () { eval $1=\$$1\$2 - } + } ;; +esac fi # as_fn_append # as_fn_arith ARG... @@ -407,21 +404,14 @@ then : { as_val=$(( $* )) }' -else $as_nop - as_fn_arith () +else case e in #( + e) as_fn_arith () { as_val=`expr "$@" || test $? -eq 1` - } + } ;; +esac fi # as_fn_arith -# as_fn_nop -# --------- -# Do nothing but, unlike ":", preserve the value of $?. -as_fn_nop () -{ - return $? -} -as_nop=as_fn_nop # as_fn_error STATUS ERROR [LINENO LOG_FD] # ---------------------------------------- @@ -495,6 +485,8 @@ as_cr_alnum=$as_cr_Letters$as_cr_digits /[$]LINENO/= ' <$as_myself | sed ' + t clear + :clear s/[$]LINENO.*/&-/ t lineno b @@ -543,7 +535,6 @@ esac as_echo='printf %s\n' as_echo_n='printf %s' - rm -f conf$$ conf$$.exe conf$$.file if test -d conf$$.dir; then rm -f conf$$.dir/conf$$.file @@ -555,9 +546,9 @@ if (echo >conf$$.file) 2>/dev/null; then if ln -s conf$$.file conf$$ 2>/dev/null; then as_ln_s='ln -s' # ... but there are two gotchas: - # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. - # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. - # In both cases, we have to default to `cp -pR'. + # 1) On MSYS, both 'ln -s file dir' and 'ln file dir' fail. + # 2) DJGPP < 2.04 has no symlinks; 'ln -s' creates a wrapper executable. + # In both cases, we have to default to 'cp -pR'. ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || as_ln_s='cp -pR' elif ln conf$$.file conf$$ 2>/dev/null; then @@ -582,10 +573,12 @@ as_test_x='test -x' as_executable_p=as_fn_executable_p # Sed expression to map a string onto a valid CPP name. -as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" +as_sed_cpp="y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g" +as_tr_cpp="eval sed '$as_sed_cpp'" # deprecated # Sed expression to map a string onto a valid variable name. -as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" +as_sed_sh="y%*+%pp%;s%[^_$as_cr_alnum]%_%g" +as_tr_sh="eval sed '$as_sed_sh'" # deprecated test -n "$DJDIR" || exec 7<&0 /dev/null && - as_fn_error $? "invalid feature name: \`$ac_useropt'" + as_fn_error $? "invalid feature name: '$ac_useropt'" ac_useropt_orig=$ac_useropt ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1313,7 +1306,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid feature name: \`$ac_useropt'" + as_fn_error $? "invalid feature name: '$ac_useropt'" ac_useropt_orig=$ac_useropt ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1526,7 +1519,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid package name: \`$ac_useropt'" + as_fn_error $? "invalid package name: '$ac_useropt'" ac_useropt_orig=$ac_useropt ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1542,7 +1535,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid package name: \`$ac_useropt'" + as_fn_error $? "invalid package name: '$ac_useropt'" ac_useropt_orig=$ac_useropt ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1572,8 +1565,8 @@ do | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) x_libraries=$ac_optarg ;; - -*) as_fn_error $? "unrecognized option: \`$ac_option' -Try \`$0 --help' for more information" + -*) as_fn_error $? "unrecognized option: '$ac_option' +Try '$0 --help' for more information" ;; *=*) @@ -1581,7 +1574,7 @@ Try \`$0 --help' for more information" # Reject names that are not valid shell variable names. case $ac_envvar in #( '' | [0-9]* | *[!_$as_cr_alnum]* ) - as_fn_error $? "invalid variable name: \`$ac_envvar'" ;; + as_fn_error $? "invalid variable name: '$ac_envvar'" ;; esac eval $ac_envvar=\$ac_optarg export $ac_envvar ;; @@ -1631,7 +1624,7 @@ do as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val" done -# There might be people who depend on the old broken behavior: `$host' +# There might be people who depend on the old broken behavior: '$host' # used to hold the argument of --host etc. # FIXME: To remove some day. build=$build_alias @@ -1699,7 +1692,7 @@ if test ! -r "$srcdir/$ac_unique_file"; then test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir" fi -ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" +ac_msg="sources are in $srcdir, but 'cd $srcdir' does not work" ac_abs_confdir=`( cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg" pwd)` @@ -1727,7 +1720,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures python 3.14 to adapt to many kinds of systems. +'configure' configures python 3.14 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1741,11 +1734,11 @@ Configuration: --help=short display options specific to this package --help=recursive display the short help of all the included packages -V, --version display version information and exit - -q, --quiet, --silent do not print \`checking ...' messages + -q, --quiet, --silent do not print 'checking ...' messages --cache-file=FILE cache test results in FILE [disabled] - -C, --config-cache alias for \`--cache-file=config.cache' + -C, --config-cache alias for '--cache-file=config.cache' -n, --no-create do not create output files - --srcdir=DIR find the sources in DIR [configure dir or \`..'] + --srcdir=DIR find the sources in DIR [configure dir or '..'] Installation directories: --prefix=PREFIX install architecture-independent files in PREFIX @@ -1753,10 +1746,10 @@ Installation directories: --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX [PREFIX] -By default, \`make install' will install all the files in -\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify -an installation prefix other than \`$ac_default_prefix' using \`--prefix', -for instance \`--prefix=\$HOME'. +By default, 'make install' will install all the files in +'$ac_default_prefix/bin', '$ac_default_prefix/lib' etc. You can specify +an installation prefix other than '$ac_default_prefix' using '--prefix', +for instance '--prefix=\$HOME'. For better control, use the options below. @@ -2020,7 +2013,7 @@ Some influential environment variables: C compiler flags for PANEL, overriding pkg-config PANEL_LIBS linker flags for PANEL, overriding pkg-config -Use these variables to override the choices made by `configure' or to help +Use these variables to override the choices made by 'configure' or to help it to find libraries and programs with nonstandard names/locations. Report bugs to . @@ -2088,9 +2081,9 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF python configure 3.14 -generated by GNU Autoconf 2.71 +generated by GNU Autoconf 2.72 -Copyright (C) 2021 Free Software Foundation, Inc. +Copyright (C) 2023 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. _ACEOF @@ -2129,11 +2122,12 @@ printf "%s\n" "$ac_try_echo"; } >&5 } && test -s conftest.$ac_objext then : ac_retval=0 -else $as_nop - printf "%s\n" "$as_me: failed program was:" >&5 +else case e in #( + e) printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 - ac_retval=1 + ac_retval=1 ;; +esac fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval @@ -2167,11 +2161,12 @@ printf "%s\n" "$ac_try_echo"; } >&5 } then : ac_retval=0 -else $as_nop - printf "%s\n" "$as_me: failed program was:" >&5 +else case e in #( + e) printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 - ac_retval=1 + ac_retval=1 ;; +esac fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval @@ -2190,8 +2185,8 @@ printf %s "checking for $2... " >&6; } if eval test \${$3+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 #include <$2> @@ -2199,10 +2194,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : eval "$3=yes" -else $as_nop - eval "$3=no" +else case e in #( + e) eval "$3=no" ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi eval ac_res=\$$3 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -2242,11 +2239,12 @@ printf "%s\n" "$ac_try_echo"; } >&5 } then : ac_retval=0 -else $as_nop - printf "%s\n" "$as_me: failed program was:" >&5 +else case e in #( + e) printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 - ac_retval=1 + ac_retval=1 ;; +esac fi # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would @@ -2288,12 +2286,13 @@ printf "%s\n" "$ac_try_echo"; } >&5 test $ac_status = 0; }; } then : ac_retval=0 -else $as_nop - printf "%s\n" "$as_me: program exited with status $ac_status" >&5 +else case e in #( + e) printf "%s\n" "$as_me: program exited with status $ac_status" >&5 printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 - ac_retval=$ac_status + ac_retval=$ac_status ;; +esac fi rm -rf conftest.dSYM conftest_ipa8_conftest.oo eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno @@ -2313,8 +2312,8 @@ printf %s "checking for $2... " >&6; } if eval test \${$3+y} then : printf %s "(cached) " >&6 -else $as_nop - eval "$3=no" +else case e in #( + e) eval "$3=no" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 @@ -2344,12 +2343,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : -else $as_nop - eval "$3=yes" +else case e in #( + e) eval "$3=yes" ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi eval ac_res=\$$3 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -2403,18 +2404,19 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_hi=$ac_mid; break -else $as_nop - as_fn_arith $ac_mid + 1 && ac_lo=$as_val +else case e in #( + e) as_fn_arith $ac_mid + 1 && ac_lo=$as_val if test $ac_lo -le $ac_mid; then ac_lo= ac_hi= break fi - as_fn_arith 2 '*' $ac_mid + 1 && ac_mid=$as_val + as_fn_arith 2 '*' $ac_mid + 1 && ac_mid=$as_val ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext done -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int @@ -2449,20 +2451,23 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_lo=$ac_mid; break -else $as_nop - as_fn_arith '(' $ac_mid ')' - 1 && ac_hi=$as_val +else case e in #( + e) as_fn_arith '(' $ac_mid ')' - 1 && ac_hi=$as_val if test $ac_mid -le $ac_hi; then ac_lo= ac_hi= break fi - as_fn_arith 2 '*' $ac_mid && ac_mid=$as_val + as_fn_arith 2 '*' $ac_mid && ac_mid=$as_val ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext done -else $as_nop - ac_lo= ac_hi= +else case e in #( + e) ac_lo= ac_hi= ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext # Binary search between lo and hi bounds. @@ -2485,8 +2490,9 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_hi=$ac_mid -else $as_nop - as_fn_arith '(' $ac_mid ')' + 1 && ac_lo=$as_val +else case e in #( + e) as_fn_arith '(' $ac_mid ')' + 1 && ac_lo=$as_val ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext done @@ -2534,8 +2540,9 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : echo >>conftest.val; read $3 &6; } if eval test \${$3+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Define $2 to an innocuous variant, in case declares $2. For example, HP-UX 11i declares gettimeofday. */ #define $2 innocuous_$2 /* System header to define __stub macros and hopefully few prototypes, - which can conflict with char $2 (); below. */ + which can conflict with char $2 (void); below. */ #include #undef $2 @@ -2577,7 +2584,7 @@ else $as_nop #ifdef __cplusplus extern "C" #endif -char $2 (); +char $2 (void); /* The GNU C library defines this for functions which it implements to always fail with ENOSYS. Some functions are actually named something starting with __ and the normal name is an alias. */ @@ -2596,11 +2603,13 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : eval "$3=yes" -else $as_nop - eval "$3=no" +else case e in #( + e) eval "$3=no" ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext + conftest$ac_exeext conftest.$ac_ext ;; +esac fi eval ac_res=\$$3 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -2622,8 +2631,8 @@ printf %s "checking whether $as_decl_name is declared... " >&6; } if eval test \${$3+y} then : printf %s "(cached) " >&6 -else $as_nop - as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'` +else case e in #( + e) as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'` eval ac_save_FLAGS=\$$6 as_fn_append $6 " $5" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -2647,12 +2656,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : eval "$3=yes" -else $as_nop - eval "$3=no" +else case e in #( + e) eval "$3=no" ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext eval $6=\$ac_save_FLAGS - + ;; +esac fi eval ac_res=\$$3 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -2673,8 +2684,8 @@ printf %s "checking for $2.$3... " >&6; } if eval test \${$4+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $5 int @@ -2690,8 +2701,8 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : eval "$4=yes" -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $5 int @@ -2707,12 +2718,15 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : eval "$4=yes" -else $as_nop - eval "$4=no" +else case e in #( + e) eval "$4=no" ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi eval ac_res=\$$4 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -2745,7 +2759,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. It was created by python $as_me 3.14, which was -generated by GNU Autoconf 2.71. Invocation command line was +generated by GNU Autoconf 2.72. Invocation command line was $ $0$ac_configure_args_raw @@ -2991,10 +3005,10 @@ esac printf "%s\n" "$as_me: loading site script $ac_site_file" >&6;} sed 's/^/| /' "$ac_site_file" >&5 . "$ac_site_file" \ - || { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} + || { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "failed to load site script $ac_site_file -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } fi done @@ -3030,9 +3044,7 @@ struct stat; /* Most of the following tests are stolen from RCS 5.7 src/conf.sh. */ struct buf { int x; }; struct buf * (*rcsopen) (struct buf *, struct stat *, int); -static char *e (p, i) - char **p; - int i; +static char *e (char **p, int i) { return p[i]; } @@ -3046,6 +3058,21 @@ static char *f (char * (*g) (char **, int), char **p, ...) return s; } +/* C89 style stringification. */ +#define noexpand_stringify(a) #a +const char *stringified = noexpand_stringify(arbitrary+token=sequence); + +/* C89 style token pasting. Exercises some of the corner cases that + e.g. old MSVC gets wrong, but not very hard. */ +#define noexpand_concat(a,b) a##b +#define expand_concat(a,b) noexpand_concat(a,b) +extern int vA; +extern int vbee; +#define aye A +#define bee B +int *pvA = &expand_concat(v,aye); +int *pvbee = &noexpand_concat(v,bee); + /* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has function prototypes and stuff, but not \xHH hex character constants. These do not provoke an error unfortunately, instead are silently treated @@ -3073,16 +3100,19 @@ ok |= (argc == 0 || f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]); # Test code for whether the C compiler supports C99 (global declarations) ac_c_conftest_c99_globals=' -// Does the compiler advertise C99 conformance? +/* Does the compiler advertise C99 conformance? */ #if !defined __STDC_VERSION__ || __STDC_VERSION__ < 199901L # error "Compiler does not advertise C99 conformance" #endif +// See if C++-style comments work. + #include extern int puts (const char *); extern int printf (const char *, ...); extern int dprintf (int, const char *, ...); extern void *malloc (size_t); +extern void free (void *); // Check varargs macros. These examples are taken from C99 6.10.3.5. // dprintf is used instead of fprintf to avoid needing to declare @@ -3132,7 +3162,6 @@ typedef const char *ccp; static inline int test_restrict (ccp restrict text) { - // See if C++-style comments work. // Iterate through items via the restricted pointer. // Also check for declarations in for loops. for (unsigned int i = 0; *(text+i) != '\''\0'\''; ++i) @@ -3198,6 +3227,8 @@ ac_c_conftest_c99_main=' ia->datasize = 10; for (int i = 0; i < ia->datasize; ++i) ia->data[i] = i * 1.234; + // Work around memory leak warnings. + free (ia); // Check named initializers. struct named_init ni = { @@ -3219,7 +3250,7 @@ ac_c_conftest_c99_main=' # Test code for whether the C compiler supports C11 (global declarations) ac_c_conftest_c11_globals=' -// Does the compiler advertise C11 conformance? +/* Does the compiler advertise C11 conformance? */ #if !defined __STDC_VERSION__ || __STDC_VERSION__ < 201112L # error "Compiler does not advertise C11 conformance" #endif @@ -3413,8 +3444,9 @@ IFS=$as_save_IFS if $as_found then : -else $as_nop - as_fn_error $? "cannot find required auxiliary files:$ac_missing_aux_files" "$LINENO" 5 +else case e in #( + e) as_fn_error $? "cannot find required auxiliary files:$ac_missing_aux_files" "$LINENO" 5 ;; +esac fi @@ -3442,12 +3474,12 @@ for ac_var in $ac_precious_vars; do eval ac_new_val=\$ac_env_${ac_var}_value case $ac_old_set,$ac_new_set in set,) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5 -printf "%s\n" "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: '$ac_var' was set to '$ac_old_val' in the previous run" >&5 +printf "%s\n" "$as_me: error: '$ac_var' was set to '$ac_old_val' in the previous run" >&2;} ac_cache_corrupted=: ;; ,set) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was not set in the previous run" >&5 -printf "%s\n" "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: '$ac_var' was not set in the previous run" >&5 +printf "%s\n" "$as_me: error: '$ac_var' was not set in the previous run" >&2;} ac_cache_corrupted=: ;; ,);; *) @@ -3456,18 +3488,18 @@ printf "%s\n" "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} ac_old_val_w=`echo x $ac_old_val` ac_new_val_w=`echo x $ac_new_val` if test "$ac_old_val_w" != "$ac_new_val_w"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' has changed since the previous run:" >&5 -printf "%s\n" "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: '$ac_var' has changed since the previous run:" >&5 +printf "%s\n" "$as_me: error: '$ac_var' has changed since the previous run:" >&2;} ac_cache_corrupted=: else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&5 -printf "%s\n" "$as_me: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in '$ac_var' since the previous run:" >&5 +printf "%s\n" "$as_me: warning: ignoring whitespace changes in '$ac_var' since the previous run:" >&2;} eval $ac_var=\$ac_old_val fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: former value: \`$ac_old_val'" >&5 -printf "%s\n" "$as_me: former value: \`$ac_old_val'" >&2;} - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: current value: \`$ac_new_val'" >&5 -printf "%s\n" "$as_me: current value: \`$ac_new_val'" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: former value: '$ac_old_val'" >&5 +printf "%s\n" "$as_me: former value: '$ac_old_val'" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: current value: '$ac_new_val'" >&5 +printf "%s\n" "$as_me: current value: '$ac_new_val'" >&2;} fi;; esac # Pass precious variables to config.status. @@ -3483,11 +3515,11 @@ printf "%s\n" "$as_me: current value: \`$ac_new_val'" >&2;} fi done if $ac_cache_corrupted; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 printf "%s\n" "$as_me: error: changes in the environment can compromise the build" >&2;} - as_fn_error $? "run \`${MAKE-make} distclean' and/or \`rm $cache_file' + as_fn_error $? "run '${MAKE-make} distclean' and/or 'rm $cache_file' and start over" "$LINENO" 5 fi ## -------------------- ## @@ -3538,8 +3570,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_HAS_GIT+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$HAS_GIT"; then +else case e in #( + e) if test -n "$HAS_GIT"; then ac_cv_prog_HAS_GIT="$HAS_GIT" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -3562,7 +3594,8 @@ done IFS=$as_save_IFS test -z "$ac_cv_prog_HAS_GIT" && ac_cv_prog_HAS_GIT="not-found" -fi +fi ;; +esac fi HAS_GIT=$ac_cv_prog_HAS_GIT if test -n "$HAS_GIT"; then @@ -3604,15 +3637,16 @@ printf %s "checking build system type... " >&6; } if test ${ac_cv_build+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_build_alias=$build_alias +else case e in #( + e) ac_build_alias=$build_alias test "x$ac_build_alias" = x && ac_build_alias=`$SHELL "${ac_aux_dir}config.guess"` test "x$ac_build_alias" = x && as_fn_error $? "cannot guess build type; you must specify one" "$LINENO" 5 ac_cv_build=`$SHELL "${ac_aux_dir}config.sub" $ac_build_alias` || as_fn_error $? "$SHELL ${ac_aux_dir}config.sub $ac_build_alias failed" "$LINENO" 5 - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_build" >&5 printf "%s\n" "$ac_cv_build" >&6; } @@ -3639,14 +3673,15 @@ printf %s "checking host system type... " >&6; } if test ${ac_cv_host+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "x$host_alias" = x; then +else case e in #( + e) if test "x$host_alias" = x; then ac_cv_host=$ac_cv_build else ac_cv_host=`$SHELL "${ac_aux_dir}config.sub" $host_alias` || as_fn_error $? "$SHELL ${ac_aux_dir}config.sub $host_alias failed" "$LINENO" 5 fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_host" >&5 printf "%s\n" "$ac_cv_host" >&6; } @@ -3710,8 +3745,8 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_build_python" >&5 printf "%s\n" "$with_build_python" >&6; } -else $as_nop - +else case e in #( + e) if test "x$cross_compiling" = xyes then : as_fn_error $? "Cross compiling requires --with-build-python" "$LINENO" 5 @@ -3720,7 +3755,8 @@ fi PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E' PYTHON_FOR_FREEZE="./_bootstrap_python" - + ;; +esac fi @@ -3740,15 +3776,16 @@ then : FREEZE_MODULE_DEPS='$(FREEZE_MODULE_BOOTSTRAP_DEPS)' PYTHON_FOR_BUILD_DEPS='' -else $as_nop - +else case e in #( + e) FREEZE_MODULE_BOOTSTRAP='./Programs/_freeze_module' FREEZE_MODULE_BOOTSTRAP_DEPS="Programs/_freeze_module" FREEZE_MODULE='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' FREEZE_MODULE_DEPS="_bootstrap_python \$(srcdir)/Programs/_freeze_module.py" PYTHON_FOR_BUILD_DEPS='$(BUILDPYTHON)' - + ;; +esac fi @@ -3765,8 +3802,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_PYTHON_FOR_REGEN+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$PYTHON_FOR_REGEN"; then +else case e in #( + e) if test -n "$PYTHON_FOR_REGEN"; then ac_cv_prog_PYTHON_FOR_REGEN="$PYTHON_FOR_REGEN" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -3788,7 +3825,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi PYTHON_FOR_REGEN=$ac_cv_prog_PYTHON_FOR_REGEN if test -n "$PYTHON_FOR_REGEN"; then @@ -3870,9 +3908,10 @@ CONFIG_ARGS="$ac_configure_args" if test ${with_pkg_config+y} then : withval=$with_pkg_config; -else $as_nop - with_pkg_config=check - +else case e in #( + e) with_pkg_config=check + ;; +esac fi case $with_pkg_config in #( @@ -3899,8 +3938,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_PKG_CONFIG+y} then : printf %s "(cached) " >&6 -else $as_nop - case $PKG_CONFIG in +else case e in #( + e) case $PKG_CONFIG in [\\/]* | ?:[\\/]*) ac_cv_path_PKG_CONFIG="$PKG_CONFIG" # Let the user override the test with a path. ;; @@ -3925,6 +3964,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi PKG_CONFIG=$ac_cv_path_PKG_CONFIG @@ -3947,8 +3987,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_PKG_CONFIG+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_PKG_CONFIG in +else case e in #( + e) case $ac_pt_PKG_CONFIG in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_PKG_CONFIG="$ac_pt_PKG_CONFIG" # Let the user override the test with a path. ;; @@ -3973,6 +4013,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_PKG_CONFIG=$ac_cv_path_ac_pt_PKG_CONFIG @@ -4208,11 +4249,12 @@ then : esac -else $as_nop - +else case e in #( + e) UNIVERSALSDK= enable_universalsdk= - + ;; +esac fi if test -n "${UNIVERSALSDK}" @@ -4273,12 +4315,13 @@ then : PYTHONFRAMEWORKDIR=${withval}.framework PYTHONFRAMEWORKIDENTIFIER=org.python.`echo $withval | tr 'A-Z' 'a-z'` -else $as_nop - +else case e in #( + e) PYTHONFRAMEWORK=Python PYTHONFRAMEWORKDIR=Python.framework PYTHONFRAMEWORKIDENTIFIER=org.python.python - + ;; +esac fi # Check whether --enable-framework was given. @@ -4411,8 +4454,8 @@ then : esac esac -else $as_nop - +else case e in #( + e) case $ac_sys_system in iOS) as_fn_error $? "iOS builds must use --enable-framework" "$LINENO" 5 ;; *) @@ -4435,7 +4478,8 @@ else $as_nop fi enable_framework= esac - + ;; +esac fi @@ -4484,8 +4528,8 @@ printf "%s\n" "applying custom app store compliance patch" >&6; } ;; esac -else $as_nop - +else case e in #( + e) case $ac_sys_system in iOS) # Always apply the compliance patch on iOS; we can use the macOS patch @@ -4500,7 +4544,8 @@ printf "%s\n" "applying default app store compliance patch" >&6; } printf "%s\n" "not patching for app store compliance" >&6; } ;; esac - + ;; +esac fi @@ -4742,8 +4787,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_HAS_XCRUN+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$HAS_XCRUN"; then +else case e in #( + e) if test -n "$HAS_XCRUN"; then ac_cv_prog_HAS_XCRUN="$HAS_XCRUN" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4766,7 +4811,8 @@ done IFS=$as_save_IFS test -z "$ac_cv_prog_HAS_XCRUN" && ac_cv_prog_HAS_XCRUN="missing" -fi +fi ;; +esac fi HAS_XCRUN=$ac_cv_prog_HAS_XCRUN if test -n "$HAS_XCRUN"; then @@ -4869,8 +4915,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CC"; then +else case e in #( + e) if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4892,7 +4938,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi CC=$ac_cv_prog_CC if test -n "$CC"; then @@ -4914,8 +4961,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_CC"; then +else case e in #( + e) if test -n "$ac_ct_CC"; then ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4937,7 +4984,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_CC=$ac_cv_prog_ac_ct_CC if test -n "$ac_ct_CC"; then @@ -4972,8 +5020,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CC"; then +else case e in #( + e) if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4995,7 +5043,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi CC=$ac_cv_prog_CC if test -n "$CC"; then @@ -5017,8 +5066,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CC"; then +else case e in #( + e) if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else ac_prog_rejected=no @@ -5057,7 +5106,8 @@ if test $ac_prog_rejected = yes; then ac_cv_prog_CC="$as_dir$ac_word${1+' '}$@" fi fi -fi +fi ;; +esac fi CC=$ac_cv_prog_CC if test -n "$CC"; then @@ -5081,8 +5131,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CC"; then +else case e in #( + e) if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -5104,7 +5154,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi CC=$ac_cv_prog_CC if test -n "$CC"; then @@ -5130,8 +5181,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_CC"; then +else case e in #( + e) if test -n "$ac_ct_CC"; then ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -5153,7 +5204,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_CC=$ac_cv_prog_ac_ct_CC if test -n "$ac_ct_CC"; then @@ -5191,8 +5243,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CC"; then +else case e in #( + e) if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -5214,7 +5266,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi CC=$ac_cv_prog_CC if test -n "$CC"; then @@ -5236,8 +5289,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_CC"; then +else case e in #( + e) if test -n "$ac_ct_CC"; then ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -5259,7 +5312,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_CC=$ac_cv_prog_ac_ct_CC if test -n "$ac_ct_CC"; then @@ -5288,10 +5342,10 @@ fi fi -test -z "$CC" && { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +test -z "$CC" && { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "no acceptable C compiler found in \$PATH -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } # Provide some information about the compiler. printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 @@ -5363,8 +5417,8 @@ printf "%s\n" "$ac_try_echo"; } >&5 printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } then : - # Autoconf-2.13 could set the ac_cv_exeext variable to `no'. -# So ignore a value of `no', otherwise this would lead to `EXEEXT = no' + # Autoconf-2.13 could set the ac_cv_exeext variable to 'no'. +# So ignore a value of 'no', otherwise this would lead to 'EXEEXT = no' # in a Makefile. We should not override ac_cv_exeext if it was cached, # so that the user can short-circuit this test for compilers unknown to # Autoconf. @@ -5384,7 +5438,7 @@ do ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` fi # We set ac_cv_exeext here because the later test for it is not - # safe: cross compilers may not add the suffix if given an `-o' + # safe: cross compilers may not add the suffix if given an '-o' # argument, so we may need to know it at that point already. # Even if this section looks crufty: it has the advantage of # actually working. @@ -5395,8 +5449,9 @@ do done test "$ac_cv_exeext" = no && ac_cv_exeext= -else $as_nop - ac_file='' +else case e in #( + e) ac_file='' ;; +esac fi if test -z "$ac_file" then : @@ -5405,13 +5460,14 @@ printf "%s\n" "no" >&6; } printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 -{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "C compiler cannot create executables -See \`config.log' for more details" "$LINENO" 5; } -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } +See 'config.log' for more details" "$LINENO" 5; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5 printf %s "checking for C compiler default output file name... " >&6; } @@ -5435,10 +5491,10 @@ printf "%s\n" "$ac_try_echo"; } >&5 printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } then : - # If both `conftest.exe' and `conftest' are `present' (well, observable) -# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will -# work properly (i.e., refer to `conftest.exe'), while it won't with -# `rm'. + # If both 'conftest.exe' and 'conftest' are 'present' (well, observable) +# catch 'conftest.exe'. For instance with Cygwin, 'ls conftest' will +# work properly (i.e., refer to 'conftest.exe'), while it won't with +# 'rm'. for ac_file in conftest.exe conftest conftest.*; do test -f "$ac_file" || continue case $ac_file in @@ -5448,11 +5504,12 @@ for ac_file in conftest.exe conftest conftest.*; do * ) break;; esac done -else $as_nop - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "cannot compute suffix of executables: cannot compile and link -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } ;; +esac fi rm -f conftest conftest$ac_cv_exeext { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 @@ -5468,6 +5525,8 @@ int main (void) { FILE *f = fopen ("conftest.out", "w"); + if (!f) + return 1; return ferror (f) || fclose (f) != 0; ; @@ -5507,26 +5566,27 @@ printf "%s\n" "$ac_try_echo"; } >&5 if test "$cross_compiling" = maybe; then cross_compiling=yes else - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot run C compiled programs. -If you meant to cross compile, use \`--host'. -See \`config.log' for more details" "$LINENO" 5; } +If you meant to cross compile, use '--host'. +See 'config.log' for more details" "$LINENO" 5; } fi fi fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5 printf "%s\n" "$cross_compiling" >&6; } -rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out +rm -f conftest.$ac_ext conftest$ac_cv_exeext \ + conftest.o conftest.obj conftest.out ac_clean_files=$ac_clean_files_save { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 printf %s "checking for suffix of object files... " >&6; } if test ${ac_cv_objext+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -5558,16 +5618,18 @@ then : break;; esac done -else $as_nop - printf "%s\n" "$as_me: failed program was:" >&5 +else case e in #( + e) printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 -{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "cannot compute suffix of object files: cannot compile -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } ;; +esac fi -rm -f conftest.$ac_cv_objext conftest.$ac_ext +rm -f conftest.$ac_cv_objext conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5 printf "%s\n" "$ac_cv_objext" >&6; } @@ -5578,8 +5640,8 @@ printf %s "checking whether the compiler supports GNU C... " >&6; } if test ${ac_cv_c_compiler_gnu+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -5596,12 +5658,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_compiler_gnu=yes -else $as_nop - ac_compiler_gnu=no +else case e in #( + e) ac_compiler_gnu=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ac_cv_c_compiler_gnu=$ac_compiler_gnu - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5 printf "%s\n" "$ac_cv_c_compiler_gnu" >&6; } @@ -5619,8 +5683,8 @@ printf %s "checking whether $CC accepts -g... " >&6; } if test ${ac_cv_prog_cc_g+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_c_werror_flag=$ac_c_werror_flag +else case e in #( + e) ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes ac_cv_prog_cc_g=no CFLAGS="-g" @@ -5638,8 +5702,8 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_prog_cc_g=yes -else $as_nop - CFLAGS="" +else case e in #( + e) CFLAGS="" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5654,8 +5718,8 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : -else $as_nop - ac_c_werror_flag=$ac_save_c_werror_flag +else case e in #( + e) ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="-g" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5672,12 +5736,15 @@ if ac_fn_c_try_compile "$LINENO" then : ac_cv_prog_cc_g=yes fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - ac_c_werror_flag=$ac_save_c_werror_flag + ac_c_werror_flag=$ac_save_c_werror_flag ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5 printf "%s\n" "$ac_cv_prog_cc_g" >&6; } @@ -5704,8 +5771,8 @@ printf %s "checking for $CC option to enable C11 features... " >&6; } if test ${ac_cv_prog_cc_c11+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_prog_cc_c11=no +else case e in #( + e) ac_cv_prog_cc_c11=no ac_save_CC=$CC cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5722,25 +5789,28 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam test "x$ac_cv_prog_cc_c11" != "xno" && break done rm -f conftest.$ac_ext -CC=$ac_save_CC +CC=$ac_save_CC ;; +esac fi if test "x$ac_cv_prog_cc_c11" = xno then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 printf "%s\n" "unsupported" >&6; } -else $as_nop - if test "x$ac_cv_prog_cc_c11" = x +else case e in #( + e) if test "x$ac_cv_prog_cc_c11" = x then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 printf "%s\n" "none needed" >&6; } -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c11" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c11" >&5 printf "%s\n" "$ac_cv_prog_cc_c11" >&6; } - CC="$CC $ac_cv_prog_cc_c11" + CC="$CC $ac_cv_prog_cc_c11" ;; +esac fi ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c11 - ac_prog_cc_stdc=c11 + ac_prog_cc_stdc=c11 ;; +esac fi fi if test x$ac_prog_cc_stdc = xno @@ -5750,8 +5820,8 @@ printf %s "checking for $CC option to enable C99 features... " >&6; } if test ${ac_cv_prog_cc_c99+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_prog_cc_c99=no +else case e in #( + e) ac_cv_prog_cc_c99=no ac_save_CC=$CC cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5768,25 +5838,28 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam test "x$ac_cv_prog_cc_c99" != "xno" && break done rm -f conftest.$ac_ext -CC=$ac_save_CC +CC=$ac_save_CC ;; +esac fi if test "x$ac_cv_prog_cc_c99" = xno then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 printf "%s\n" "unsupported" >&6; } -else $as_nop - if test "x$ac_cv_prog_cc_c99" = x +else case e in #( + e) if test "x$ac_cv_prog_cc_c99" = x then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 printf "%s\n" "none needed" >&6; } -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c99" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c99" >&5 printf "%s\n" "$ac_cv_prog_cc_c99" >&6; } - CC="$CC $ac_cv_prog_cc_c99" + CC="$CC $ac_cv_prog_cc_c99" ;; +esac fi ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c99 - ac_prog_cc_stdc=c99 + ac_prog_cc_stdc=c99 ;; +esac fi fi if test x$ac_prog_cc_stdc = xno @@ -5796,8 +5869,8 @@ printf %s "checking for $CC option to enable C89 features... " >&6; } if test ${ac_cv_prog_cc_c89+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_prog_cc_c89=no +else case e in #( + e) ac_cv_prog_cc_c89=no ac_save_CC=$CC cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5814,25 +5887,28 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam test "x$ac_cv_prog_cc_c89" != "xno" && break done rm -f conftest.$ac_ext -CC=$ac_save_CC +CC=$ac_save_CC ;; +esac fi if test "x$ac_cv_prog_cc_c89" = xno then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 printf "%s\n" "unsupported" >&6; } -else $as_nop - if test "x$ac_cv_prog_cc_c89" = x +else case e in #( + e) if test "x$ac_cv_prog_cc_c89" = x then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 printf "%s\n" "none needed" >&6; } -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 printf "%s\n" "$ac_cv_prog_cc_c89" >&6; } - CC="$CC $ac_cv_prog_cc_c89" + CC="$CC $ac_cv_prog_cc_c89" ;; +esac fi ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c89 - ac_prog_cc_stdc=c89 + ac_prog_cc_stdc=c89 ;; +esac fi fi @@ -5857,8 +5933,8 @@ if test -z "$CPP"; then if test ${ac_cv_prog_CPP+y} then : printf %s "(cached) " >&6 -else $as_nop - # Double quotes because $CC needs to be expanded +else case e in #( + e) # Double quotes because $CC needs to be expanded for CPP in "$CC -E" "$CC -E -traditional-cpp" cpp /lib/cpp do ac_preproc_ok=false @@ -5876,9 +5952,10 @@ _ACEOF if ac_fn_c_try_cpp "$LINENO" then : -else $as_nop - # Broken: fails on valid input. -continue +else case e in #( + e) # Broken: fails on valid input. +continue ;; +esac fi rm -f conftest.err conftest.i conftest.$ac_ext @@ -5892,15 +5969,16 @@ if ac_fn_c_try_cpp "$LINENO" then : # Broken: success on invalid input. continue -else $as_nop - # Passes both tests. +else case e in #( + e) # Passes both tests. ac_preproc_ok=: -break +break ;; +esac fi rm -f conftest.err conftest.i conftest.$ac_ext done -# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +# Because of 'break', _AC_PREPROC_IFELSE's cleaning code was skipped. rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok then : @@ -5909,7 +5987,8 @@ fi done ac_cv_prog_CPP=$CPP - + ;; +esac fi CPP=$ac_cv_prog_CPP else @@ -5932,9 +6011,10 @@ _ACEOF if ac_fn_c_try_cpp "$LINENO" then : -else $as_nop - # Broken: fails on valid input. -continue +else case e in #( + e) # Broken: fails on valid input. +continue ;; +esac fi rm -f conftest.err conftest.i conftest.$ac_ext @@ -5948,24 +6028,26 @@ if ac_fn_c_try_cpp "$LINENO" then : # Broken: success on invalid input. continue -else $as_nop - # Passes both tests. +else case e in #( + e) # Passes both tests. ac_preproc_ok=: -break +break ;; +esac fi rm -f conftest.err conftest.i conftest.$ac_ext done -# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +# Because of 'break', _AC_PREPROC_IFELSE's cleaning code was skipped. rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok then : -else $as_nop - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "C preprocessor \"$CPP\" fails sanity check -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } ;; +esac fi ac_ext=c @@ -5979,8 +6061,8 @@ printf %s "checking for grep that handles long lines and -e... " >&6; } if test ${ac_cv_path_GREP+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -z "$GREP"; then +else case e in #( + e) if test -z "$GREP"; then ac_path_GREP_found=false # Loop through the user's path and test for each of PROGNAME-LIST as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -5999,9 +6081,10 @@ do as_fn_executable_p "$ac_path_GREP" || continue # Check for GNU ac_path_GREP and select it if it is found. # Check for GNU $ac_path_GREP -case `"$ac_path_GREP" --version 2>&1` in +case `"$ac_path_GREP" --version 2>&1` in #( *GNU*) ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;; +#( *) ac_count=0 printf %s 0123456789 >"conftest.in" @@ -6036,7 +6119,8 @@ IFS=$as_save_IFS else ac_cv_path_GREP=$GREP fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_GREP" >&5 printf "%s\n" "$ac_cv_path_GREP" >&6; } @@ -6048,8 +6132,8 @@ printf %s "checking for a sed that does not truncate output... " >&6; } if test ${ac_cv_path_SED+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ +else case e in #( + e) ac_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ for ac_i in 1 2 3 4 5 6 7; do ac_script="$ac_script$as_nl$ac_script" done @@ -6074,9 +6158,10 @@ do as_fn_executable_p "$ac_path_SED" || continue # Check for GNU ac_path_SED and select it if it is found. # Check for GNU $ac_path_SED -case `"$ac_path_SED" --version 2>&1` in +case `"$ac_path_SED" --version 2>&1` in #( *GNU*) ac_cv_path_SED="$ac_path_SED" ac_path_SED_found=:;; +#( *) ac_count=0 printf %s 0123456789 >"conftest.in" @@ -6111,7 +6196,8 @@ IFS=$as_save_IFS else ac_cv_path_SED=$SED fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_SED" >&5 printf "%s\n" "$ac_cv_path_SED" >&6; } @@ -6123,8 +6209,8 @@ printf %s "checking for egrep... " >&6; } if test ${ac_cv_path_EGREP+y} then : printf %s "(cached) " >&6 -else $as_nop - if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 +else case e in #( + e) if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 then ac_cv_path_EGREP="$GREP -E" else if test -z "$EGREP"; then @@ -6146,9 +6232,10 @@ do as_fn_executable_p "$ac_path_EGREP" || continue # Check for GNU ac_path_EGREP and select it if it is found. # Check for GNU $ac_path_EGREP -case `"$ac_path_EGREP" --version 2>&1` in +case `"$ac_path_EGREP" --version 2>&1` in #( *GNU*) ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; +#( *) ac_count=0 printf %s 0123456789 >"conftest.in" @@ -6184,12 +6271,15 @@ else ac_cv_path_EGREP=$EGREP fi - fi + fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5 printf "%s\n" "$ac_cv_path_EGREP" >&6; } EGREP="$ac_cv_path_EGREP" + EGREP_TRADITIONAL=$EGREP + ac_cv_path_EGREP_TRADITIONAL=$EGREP CC_BASENAME=$(expr "//$CC" : '.*/\(.*\)') @@ -6199,8 +6289,8 @@ printf %s "checking for CC compiler name... " >&6; } if test ${ac_cv_cc_name+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat > conftest.c <&5 printf "%s\n" "$ac_cv_cc_name" >&6; } @@ -6278,8 +6369,8 @@ printf %s "checking whether it is safe to define __EXTENSIONS__... " >&6; } if test ${ac_cv_safe_to_define___extensions__+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ # define __EXTENSIONS__ 1 @@ -6295,10 +6386,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_safe_to_define___extensions__=yes -else $as_nop - ac_cv_safe_to_define___extensions__=no +else case e in #( + e) ac_cv_safe_to_define___extensions__=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_safe_to_define___extensions__" >&5 printf "%s\n" "$ac_cv_safe_to_define___extensions__" >&6; } @@ -6308,8 +6401,8 @@ printf %s "checking whether _XOPEN_SOURCE should be defined... " >&6; } if test ${ac_cv_should_define__xopen_source+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_should_define__xopen_source=no +else case e in #( + e) ac_cv_should_define__xopen_source=no if test $ac_cv_header_wchar_h = yes then : cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -6328,8 +6421,8 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define _XOPEN_SOURCE 500 @@ -6347,10 +6440,12 @@ if ac_fn_c_try_compile "$LINENO" then : ac_cv_should_define__xopen_source=yes fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext -fi +fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_should_define__xopen_source" >&5 printf "%s\n" "$ac_cv_should_define__xopen_source" >&6; } @@ -6375,6 +6470,8 @@ printf "%s\n" "$ac_cv_should_define__xopen_source" >&6; } printf "%s\n" "#define __STDC_WANT_IEC_60559_DFP_EXT__ 1" >>confdefs.h + printf "%s\n" "#define __STDC_WANT_IEC_60559_EXT__ 1" >>confdefs.h + printf "%s\n" "#define __STDC_WANT_IEC_60559_FUNCS_EXT__ 1" >>confdefs.h printf "%s\n" "#define __STDC_WANT_IEC_60559_TYPES_EXT__ 1" >>confdefs.h @@ -6394,8 +6491,9 @@ then : printf "%s\n" "#define _POSIX_1_SOURCE 2" >>confdefs.h -else $as_nop - MINIX= +else case e in #( + e) MINIX= ;; +esac fi if test $ac_cv_safe_to_define___extensions__ = yes then : @@ -6415,8 +6513,8 @@ printf %s "checking for GCC compatible compiler... " >&6; } if test ${ac_cv_gcc_compat+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #if !defined(__GNUC__) @@ -6429,10 +6527,12 @@ _ACEOF if ac_fn_c_try_cpp "$LINENO" then : ac_cv_gcc_compat=yes -else $as_nop - ac_cv_gcc_compat=no +else case e in #( + e) ac_cv_gcc_compat=no ;; +esac fi -rm -f conftest.err conftest.i conftest.$ac_ext +rm -f conftest.err conftest.i conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gcc_compat" >&5 printf "%s\n" "$ac_cv_gcc_compat" >&6; } @@ -6451,8 +6551,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $CXX in +else case e in #( + e) case $CXX in [\\/]* | ?:[\\/]*) ac_cv_path_CXX="$CXX" # Let the user override the test with a path. ;; @@ -6477,6 +6577,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi CXX=$ac_cv_path_CXX @@ -6499,8 +6600,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_CXX in +else case e in #( + e) case $ac_pt_CXX in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. ;; @@ -6525,6 +6626,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_CXX=$ac_cv_path_ac_pt_CXX @@ -6559,8 +6661,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $CXX in +else case e in #( + e) case $CXX in [\\/]* | ?:[\\/]*) ac_cv_path_CXX="$CXX" # Let the user override the test with a path. ;; @@ -6585,6 +6687,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi CXX=$ac_cv_path_CXX @@ -6607,8 +6710,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_CXX in +else case e in #( + e) case $ac_pt_CXX in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. ;; @@ -6633,6 +6736,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_CXX=$ac_cv_path_ac_pt_CXX @@ -6667,8 +6771,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $CXX in +else case e in #( + e) case $CXX in [\\/]* | ?:[\\/]*) ac_cv_path_CXX="$CXX" # Let the user override the test with a path. ;; @@ -6693,6 +6797,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi CXX=$ac_cv_path_CXX @@ -6715,8 +6820,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_CXX in +else case e in #( + e) case $ac_pt_CXX in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. ;; @@ -6741,6 +6846,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_CXX=$ac_cv_path_ac_pt_CXX @@ -6775,8 +6881,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $CXX in +else case e in #( + e) case $CXX in [\\/]* | ?:[\\/]*) ac_cv_path_CXX="$CXX" # Let the user override the test with a path. ;; @@ -6801,6 +6907,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi CXX=$ac_cv_path_CXX @@ -6823,8 +6930,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_CXX in +else case e in #( + e) case $ac_pt_CXX in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. ;; @@ -6849,6 +6956,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_CXX=$ac_cv_path_ac_pt_CXX @@ -6893,8 +7001,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CXX"; then +else case e in #( + e) if test -n "$CXX"; then ac_cv_prog_CXX="$CXX" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -6916,7 +7024,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi CXX=$ac_cv_prog_CXX if test -n "$CXX"; then @@ -6942,8 +7051,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_CXX"; then +else case e in #( + e) if test -n "$ac_ct_CXX"; then ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -6965,7 +7074,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_CXX=$ac_cv_prog_ac_ct_CXX if test -n "$ac_ct_CXX"; then @@ -7139,8 +7249,8 @@ printf %s "checking for -Wl,--no-as-needed... " >&6; } if test ${ac_cv_wl_no_as_needed+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) save_LDFLAGS="$LDFLAGS" as_fn_append LDFLAGS " -Wl,--no-as-needed" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -7158,14 +7268,16 @@ if ac_fn_c_try_link "$LINENO" then : NO_AS_NEEDED="-Wl,--no-as-needed" ac_cv_wl_no_as_needed=yes -else $as_nop - NO_AS_NEEDED="" - ac_cv_wl_no_as_needed=no +else case e in #( + e) NO_AS_NEEDED="" + ac_cv_wl_no_as_needed=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext LDFLAGS="$save_LDFLAGS" - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_wl_no_as_needed" >&5 printf "%s\n" "$ac_cv_wl_no_as_needed" >&6; } @@ -7238,10 +7350,11 @@ then : ;; esac -else $as_nop - +else case e in #( + e) enable_wasm_dynamic_linking=missing - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_wasm_dynamic_linking" >&5 @@ -7263,10 +7376,11 @@ then : ;; esac -else $as_nop - +else case e in #( + e) enable_wasm_pthreads=missing - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_wasm_pthreads" >&5 @@ -7289,8 +7403,8 @@ then : ;; esac -else $as_nop - +else case e in #( + e) case $ac_sys_system in #( Emscripten) : EXEEXT=.mjs ;; #( @@ -7300,7 +7414,8 @@ else $as_nop EXEEXT= ;; esac - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $EXEEXT" >&5 @@ -7476,9 +7591,10 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; }; fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } ;; +esac fi @@ -7501,8 +7617,9 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : -else $as_nop - enable_profiling=no +else case e in #( + e) enable_profiling=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -7640,8 +7757,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_NODE+y} then : printf %s "(cached) " >&6 -else $as_nop - case $NODE in +else case e in #( + e) case $NODE in [\\/]* | ?:[\\/]*) ac_cv_path_NODE="$NODE" # Let the user override the test with a path. ;; @@ -7666,6 +7783,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi NODE=$ac_cv_path_NODE @@ -7688,8 +7806,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_NODE+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_NODE in +else case e in #( + e) case $ac_pt_NODE in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_NODE="$ac_pt_NODE" # Let the user override the test with a path. ;; @@ -7714,6 +7832,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_NODE=$ac_cv_path_ac_pt_NODE @@ -7798,8 +7917,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_AR+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$AR"; then +else case e in #( + e) if test -n "$AR"; then ac_cv_prog_AR="$AR" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -7821,7 +7940,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi AR=$ac_cv_prog_AR if test -n "$AR"; then @@ -7847,8 +7967,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_AR+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_AR"; then +else case e in #( + e) if test -n "$ac_ct_AR"; then ac_cv_prog_ac_ct_AR="$ac_ct_AR" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -7870,7 +7990,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_AR=$ac_cv_prog_ac_ct_AR if test -n "$ac_ct_AR"; then @@ -7935,8 +8056,8 @@ if test -z "$INSTALL"; then if test ${ac_cv_path_install+y} then : printf %s "(cached) " >&6 -else $as_nop - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +else case e in #( + e) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS @@ -7990,7 +8111,8 @@ esac IFS=$as_save_IFS rm -rf conftest.one conftest.two conftest.dir - + ;; +esac fi if test ${ac_cv_path_install+y}; then INSTALL=$ac_cv_path_install @@ -8020,8 +8142,8 @@ if test -z "$MKDIR_P"; then if test ${ac_cv_path_mkdir+y} then : printf %s "(cached) " >&6 -else $as_nop - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +else case e in #( + e) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH$PATH_SEPARATOR/opt/sfw/bin do IFS=$as_save_IFS @@ -8035,7 +8157,7 @@ do as_fn_executable_p "$as_dir$ac_prog$ac_exec_ext" || continue case `"$as_dir$ac_prog$ac_exec_ext" --version 2>&1` in #( 'mkdir ('*'coreutils) '* | \ - 'BusyBox '* | \ + *'BusyBox '* | \ 'mkdir (fileutils) '4.1*) ac_cv_path_mkdir=$as_dir$ac_prog$ac_exec_ext break 3;; @@ -8044,18 +8166,17 @@ do done done IFS=$as_save_IFS - + ;; +esac fi test -d ./--version && rmdir ./--version if test ${ac_cv_path_mkdir+y}; then MKDIR_P="$ac_cv_path_mkdir -p" else - # As a last resort, use the slow shell script. Don't cache a - # value for MKDIR_P within a source directory, because that will - # break other packages using the cache if that directory is - # removed, or if the value is a relative name. - MKDIR_P="$ac_install_sh -d" + # As a last resort, use plain mkdir -p, + # in the hope it doesn't have the bugs of ancient mkdir. + MKDIR_P='mkdir -p' fi fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $MKDIR_P" >&5 @@ -8087,12 +8208,14 @@ then : enableval=$enable_gil; if test "x$enable_gil" = xyes then : disable_gil=no -else $as_nop - disable_gil=yes +else case e in #( + e) disable_gil=yes ;; +esac fi -else $as_nop - disable_gil=no - +else case e in #( + e) disable_gil=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_gil" >&5 @@ -8128,9 +8251,10 @@ printf "%s\n" "yes" >&6; }; else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; }; Py_DEBUG='false' fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -8143,9 +8267,10 @@ printf %s "checking for --with-trace-refs... " >&6; } if test ${with_trace_refs+y} then : withval=$with_trace_refs; -else $as_nop - with_trace_refs=no - +else case e in #( + e) with_trace_refs=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_trace_refs" >&5 @@ -8170,9 +8295,10 @@ printf %s "checking for --enable-pystats... " >&6; } if test ${enable_pystats+y} then : enableval=$enable_pystats; -else $as_nop - enable_pystats=no - +else case e in #( + e) enable_pystats=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_pystats" >&5 @@ -8222,8 +8348,9 @@ printf %s "checking for --enable-experimental-jit... " >&6; } if test ${enable_experimental_jit+y} then : enableval=$enable_experimental_jit; -else $as_nop - enable_experimental_jit=no +else case e in #( + e) enable_experimental_jit=no ;; +esac fi case $enable_experimental_jit in @@ -8237,20 +8364,22 @@ esac if ${tier2_flags:+false} : then : -else $as_nop - as_fn_append CFLAGS_NODIST " $tier2_flags" +else case e in #( + e) as_fn_append CFLAGS_NODIST " $tier2_flags" ;; +esac fi if ${jit_flags:+false} : then : -else $as_nop - as_fn_append CFLAGS_NODIST " $jit_flags" +else case e in #( + e) as_fn_append CFLAGS_NODIST " $jit_flags" REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue then : as_fn_append REGEN_JIT_COMMAND " --debug" -fi +fi ;; +esac fi @@ -8277,9 +8406,10 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; }; fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -8299,8 +8429,8 @@ printf %s "checking whether C compiler accepts -fno-semantic-interposition... " if test ${ax_cv_check_cflags__Werror__fno_semantic_interposition+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -fno-semantic-interposition" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -8317,11 +8447,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__fno_semantic_interposition=yes -else $as_nop - ax_cv_check_cflags__Werror__fno_semantic_interposition=no +else case e in #( + e) ax_cv_check_cflags__Werror__fno_semantic_interposition=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__fno_semantic_interposition" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__fno_semantic_interposition" >&6; } @@ -8331,8 +8463,9 @@ then : CFLAGS_NODIST="$CFLAGS_NODIST -fno-semantic-interposition" LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition" -else $as_nop - : +else case e in #( + e) : ;; +esac fi @@ -8419,9 +8552,10 @@ printf "%s\n" "no" >&6; } ;; esac -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi if test "$Py_LTO" = 'true' ; then @@ -8433,8 +8567,8 @@ printf %s "checking whether C compiler accepts -flto=thin... " >&6; } if test ${ax_cv_check_cflags___flto_thin+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -flto=thin" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -8451,19 +8585,22 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags___flto_thin=yes -else $as_nop - ax_cv_check_cflags___flto_thin=no +else case e in #( + e) ax_cv_check_cflags___flto_thin=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___flto_thin" >&5 printf "%s\n" "$ax_cv_check_cflags___flto_thin" >&6; } if test "x$ax_cv_check_cflags___flto_thin" = xyes then : LDFLAGS_NOLTO="-flto=thin" -else $as_nop - LDFLAGS_NOLTO="-flto" +else case e in #( + e) LDFLAGS_NOLTO="-flto" ;; +esac fi @@ -8475,8 +8612,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_LLVM_AR+y} then : printf %s "(cached) " >&6 -else $as_nop - case $LLVM_AR in +else case e in #( + e) case $LLVM_AR in [\\/]* | ?:[\\/]*) ac_cv_path_LLVM_AR="$LLVM_AR" # Let the user override the test with a path. ;; @@ -8501,6 +8638,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi LLVM_AR=$ac_cv_path_LLVM_AR @@ -8523,8 +8661,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_LLVM_AR+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_LLVM_AR in +else case e in #( + e) case $ac_pt_LLVM_AR in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_LLVM_AR="$ac_pt_LLVM_AR" # Let the user override the test with a path. ;; @@ -8549,6 +8687,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_LLVM_AR=$ac_cv_path_ac_pt_LLVM_AR @@ -8613,8 +8752,8 @@ printf %s "checking whether C compiler accepts -flto=thin... " >&6; } if test ${ax_cv_check_cflags___flto_thin+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -flto=thin" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -8631,11 +8770,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags___flto_thin=yes -else $as_nop - ax_cv_check_cflags___flto_thin=no +else case e in #( + e) ax_cv_check_cflags___flto_thin=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___flto_thin" >&5 printf "%s\n" "$ax_cv_check_cflags___flto_thin" >&6; } @@ -8645,12 +8786,13 @@ then : LTOFLAGS="-flto=thin -Wl,-export_dynamic -Wl,-object_path_lto,\"\$@\".lto" LTOCFLAGS="-flto=thin" -else $as_nop - +else case e in #( + e) LTOFLAGS="-flto -Wl,-export_dynamic -Wl,-object_path_lto,\"\$@\".lto" LTOCFLAGS="-flto" - + ;; +esac fi else @@ -8667,8 +8809,8 @@ printf %s "checking whether C compiler accepts -flto=thin... " >&6; } if test ${ax_cv_check_cflags___flto_thin+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -flto=thin" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -8685,19 +8827,22 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags___flto_thin=yes -else $as_nop - ax_cv_check_cflags___flto_thin=no +else case e in #( + e) ax_cv_check_cflags___flto_thin=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___flto_thin" >&5 printf "%s\n" "$ax_cv_check_cflags___flto_thin" >&6; } if test "x$ax_cv_check_cflags___flto_thin" = xyes then : LTOFLAGS="-flto=thin" -else $as_nop - LTOFLAGS="-flto" +else case e in #( + e) LTOFLAGS="-flto" ;; +esac fi else @@ -8757,8 +8902,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_LLVM_PROFDATA+y} then : printf %s "(cached) " >&6 -else $as_nop - case $LLVM_PROFDATA in +else case e in #( + e) case $LLVM_PROFDATA in [\\/]* | ?:[\\/]*) ac_cv_path_LLVM_PROFDATA="$LLVM_PROFDATA" # Let the user override the test with a path. ;; @@ -8783,6 +8928,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi LLVM_PROFDATA=$ac_cv_path_LLVM_PROFDATA @@ -8805,8 +8951,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_LLVM_PROFDATA+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_LLVM_PROFDATA in +else case e in #( + e) case $ac_pt_LLVM_PROFDATA in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_LLVM_PROFDATA="$ac_pt_LLVM_PROFDATA" # Let the user override the test with a path. ;; @@ -8831,6 +8977,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_LLVM_PROFDATA=$ac_cv_path_ac_pt_LLVM_PROFDATA @@ -8927,9 +9074,10 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; }; fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -8946,8 +9094,8 @@ printf %s "checking whether C compiler accepts -fno-reorder-blocks-and-partition if test ${ax_cv_check_cflags___fno_reorder_blocks_and_partition+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -fno-reorder-blocks-and-partition" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -8964,11 +9112,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags___fno_reorder_blocks_and_partition=yes -else $as_nop - ax_cv_check_cflags___fno_reorder_blocks_and_partition=no +else case e in #( + e) ax_cv_check_cflags___fno_reorder_blocks_and_partition=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___fno_reorder_blocks_and_partition" >&5 printf "%s\n" "$ax_cv_check_cflags___fno_reorder_blocks_and_partition" >&6; } @@ -8977,8 +9127,9 @@ then : CFLAGS_NODIST="$CFLAGS_NODIST -fno-reorder-blocks-and-partition" -else $as_nop - : +else case e in #( + e) : ;; +esac fi @@ -8998,8 +9149,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_LLVM_BOLT+y} then : printf %s "(cached) " >&6 -else $as_nop - case $LLVM_BOLT in +else case e in #( + e) case $LLVM_BOLT in [\\/]* | ?:[\\/]*) ac_cv_path_LLVM_BOLT="$LLVM_BOLT" # Let the user override the test with a path. ;; @@ -9024,6 +9175,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi LLVM_BOLT=$ac_cv_path_LLVM_BOLT @@ -9046,8 +9198,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_LLVM_BOLT+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_LLVM_BOLT in +else case e in #( + e) case $ac_pt_LLVM_BOLT in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_LLVM_BOLT="$ac_pt_LLVM_BOLT" # Let the user override the test with a path. ;; @@ -9072,6 +9224,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_LLVM_BOLT=$ac_cv_path_ac_pt_LLVM_BOLT @@ -9115,8 +9268,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_MERGE_FDATA+y} then : printf %s "(cached) " >&6 -else $as_nop - case $MERGE_FDATA in +else case e in #( + e) case $MERGE_FDATA in [\\/]* | ?:[\\/]*) ac_cv_path_MERGE_FDATA="$MERGE_FDATA" # Let the user override the test with a path. ;; @@ -9141,6 +9294,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi MERGE_FDATA=$ac_cv_path_MERGE_FDATA @@ -9163,8 +9317,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_MERGE_FDATA+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_MERGE_FDATA in +else case e in #( + e) case $ac_pt_MERGE_FDATA in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_MERGE_FDATA="$ac_pt_MERGE_FDATA" # Let the user override the test with a path. ;; @@ -9189,6 +9343,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_MERGE_FDATA=$ac_cv_path_ac_pt_MERGE_FDATA @@ -9274,8 +9429,8 @@ printf %s "checking if $CC supports -fstrict-overflow and -fno-strict-overflow.. if test ${ac_cv_cc_supports_fstrict_overflow+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -9289,12 +9444,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_cc_supports_fstrict_overflow=yes -else $as_nop - ac_cv_cc_supports_fstrict_overflow=no - +else case e in #( + e) ac_cv_cc_supports_fstrict_overflow=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cc_supports_fstrict_overflow" >&5 printf "%s\n" "$ac_cv_cc_supports_fstrict_overflow" >&6; } @@ -9304,9 +9461,10 @@ if test "x$ac_cv_cc_supports_fstrict_overflow" = xyes then : STRICT_OVERFLOW_CFLAGS="-fstrict-overflow" NO_STRICT_OVERFLOW_CFLAGS="-fno-strict-overflow" -else $as_nop - STRICT_OVERFLOW_CFLAGS="" - NO_STRICT_OVERFLOW_CFLAGS="" +else case e in #( + e) STRICT_OVERFLOW_CFLAGS="" + NO_STRICT_OVERFLOW_CFLAGS="" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-strict-overflow" >&5 @@ -9322,9 +9480,10 @@ then : printf "%s\n" "$as_me: WARNING: --with-strict-overflow=yes requires a compiler that supports -fstrict-overflow" >&2;} fi -else $as_nop - with_strict_overflow=no - +else case e in #( + e) with_strict_overflow=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_strict_overflow" >&5 @@ -9338,8 +9497,8 @@ printf %s "checking if $CC supports -Og optimization level... " >&6; } if test ${ac_cv_cc_supports_og+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9357,13 +9516,15 @@ then : ac_cv_cc_supports_og=yes -else $as_nop - +else case e in #( + e) ac_cv_cc_supports_og=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cc_supports_og" >&5 printf "%s\n" "$ac_cv_cc_supports_og" >&6; } @@ -9429,8 +9590,9 @@ case $ac_sys_system in #( if test "x$Py_DEBUG" = xyes then : wasm_debug=yes -else $as_nop - wasm_debug=no +else case e in #( + e) wasm_debug=no ;; +esac fi as_fn_append LINKFORSHARED " -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520" @@ -9466,10 +9628,11 @@ then : as_fn_append LDFLAGS_NODIST " -sASSERTIONS" as_fn_append LINKFORSHARED " $WASM_LINKFORSHARED_DEBUG" -else $as_nop - +else case e in #( + e) as_fn_append LINKFORSHARED " -O2 -g0" - + ;; +esac fi ;; #( WASI) : @@ -9542,8 +9705,9 @@ UNIVERSAL_ARCH_FLAGS= if test "x$with_strict_overflow" = xyes then : BASECFLAGS="$BASECFLAGS $STRICT_OVERFLOW_CFLAGS" -else $as_nop - BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS" +else case e in #( + e) BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS" ;; +esac fi # Enable flags that warn and protect for potential security vulnerabilities. @@ -9557,11 +9721,13 @@ then : enableval=$enable_safety; if test "x$disable_safety" = xyes then : enable_safety=no -else $as_nop - enable_safety=yes +else case e in #( + e) enable_safety=yes ;; +esac fi -else $as_nop - enable_safety=no +else case e in #( + e) enable_safety=no ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_safety" >&5 @@ -9574,8 +9740,8 @@ printf %s "checking whether C compiler accepts -fstack-protector-strong... " >&6 if test ${ax_cv_check_cflags__Werror__fstack_protector_strong+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -fstack-protector-strong" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -9592,20 +9758,23 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__fstack_protector_strong=yes -else $as_nop - ax_cv_check_cflags__Werror__fstack_protector_strong=no +else case e in #( + e) ax_cv_check_cflags__Werror__fstack_protector_strong=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__fstack_protector_strong" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__fstack_protector_strong" >&6; } if test "x$ax_cv_check_cflags__Werror__fstack_protector_strong" = xyes then : CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -fstack-protector-strong not supported" >&5 -printf "%s\n" "$as_me: WARNING: -fstack-protector-strong not supported" >&2;} +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -fstack-protector-strong not supported" >&5 +printf "%s\n" "$as_me: WARNING: -fstack-protector-strong not supported" >&2;} ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wtrampolines" >&5 @@ -9613,8 +9782,8 @@ printf %s "checking whether C compiler accepts -Wtrampolines... " >&6; } if test ${ax_cv_check_cflags__Werror__Wtrampolines+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -Wtrampolines" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -9631,20 +9800,23 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__Wtrampolines=yes -else $as_nop - ax_cv_check_cflags__Werror__Wtrampolines=no +else case e in #( + e) ax_cv_check_cflags__Werror__Wtrampolines=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wtrampolines" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__Wtrampolines" >&6; } if test "x$ax_cv_check_cflags__Werror__Wtrampolines" = xyes then : CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not supported" >&5 -printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;} +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;} ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wimplicit-fallthrough" >&5 @@ -9652,8 +9824,8 @@ printf %s "checking whether C compiler accepts -Wimplicit-fallthrough... " >&6; if test ${ax_cv_check_cflags__Werror__Wimplicit_fallthrough+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -Wimplicit-fallthrough" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -9670,20 +9842,23 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__Wimplicit_fallthrough=yes -else $as_nop - ax_cv_check_cflags__Werror__Wimplicit_fallthrough=no +else case e in #( + e) ax_cv_check_cflags__Werror__Wimplicit_fallthrough=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&6; } if test "x$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" = xyes then : CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wimplicit-fallthrough not supported" >&5 -printf "%s\n" "$as_me: WARNING: -Wimplicit-fallthrough not supported" >&2;} +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wimplicit-fallthrough not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wimplicit-fallthrough not supported" >&2;} ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Werror=format-security" >&5 @@ -9691,8 +9866,8 @@ printf %s "checking whether C compiler accepts -Werror=format-security... " >&6; if test ${ax_cv_check_cflags__Werror__Werror_format_security+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -Werror=format-security" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -9709,20 +9884,23 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__Werror_format_security=yes -else $as_nop - ax_cv_check_cflags__Werror__Werror_format_security=no +else case e in #( + e) ax_cv_check_cflags__Werror__Werror_format_security=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Werror_format_security" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__Werror_format_security" >&6; } if test "x$ax_cv_check_cflags__Werror__Werror_format_security" = xyes then : CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Werror=format-security not supported" >&5 -printf "%s\n" "$as_me: WARNING: -Werror=format-security not supported" >&2;} +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Werror=format-security not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Werror=format-security not supported" >&2;} ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wbidi-chars=any" >&5 @@ -9730,8 +9908,8 @@ printf %s "checking whether C compiler accepts -Wbidi-chars=any... " >&6; } if test ${ax_cv_check_cflags__Werror__Wbidi_chars_any+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -Wbidi-chars=any" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -9748,20 +9926,23 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__Wbidi_chars_any=yes -else $as_nop - ax_cv_check_cflags__Werror__Wbidi_chars_any=no +else case e in #( + e) ax_cv_check_cflags__Werror__Wbidi_chars_any=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wbidi_chars_any" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__Wbidi_chars_any" >&6; } if test "x$ax_cv_check_cflags__Werror__Wbidi_chars_any" = xyes then : CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wbidi-chars=any not supported" >&5 -printf "%s\n" "$as_me: WARNING: -Wbidi-chars=any not supported" >&2;} +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wbidi-chars=any not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wbidi-chars=any not supported" >&2;} ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wall" >&5 @@ -9769,8 +9950,8 @@ printf %s "checking whether C compiler accepts -Wall... " >&6; } if test ${ax_cv_check_cflags__Werror__Wall+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -Wall" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -9787,20 +9968,23 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__Wall=yes -else $as_nop - ax_cv_check_cflags__Werror__Wall=no +else case e in #( + e) ax_cv_check_cflags__Werror__Wall=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wall" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__Wall" >&6; } if test "x$ax_cv_check_cflags__Werror__Wall" = xyes then : CFLAGS_NODIST="$CFLAGS_NODIST -Wall" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wall not supported" >&5 -printf "%s\n" "$as_me: WARNING: -Wall not supported" >&2;} +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wall not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wall not supported" >&2;} ;; +esac fi fi @@ -9813,11 +9997,13 @@ then : enableval=$enable_slower_safety; if test "x$disable_slower_safety" = xyes then : enable_slower_safety=no -else $as_nop - enable_slower_safety=yes +else case e in #( + e) enable_slower_safety=yes ;; +esac fi -else $as_nop - enable_slower_safety=no +else case e in #( + e) enable_slower_safety=no ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_slower_safety" >&5 @@ -9830,8 +10016,8 @@ printf %s "checking whether C compiler accepts -D_FORTIFY_SOURCE=3... " >&6; } if test ${ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -D_FORTIFY_SOURCE=3" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -9848,20 +10034,23 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3=yes -else $as_nop - ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3=no +else case e in #( + e) ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" >&6; } if test "x$ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" = xyes then : CFLAGS_NODIST="$CFLAGS_NODIST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -D_FORTIFY_SOURCE=3 not supported" >&5 -printf "%s\n" "$as_me: WARNING: -D_FORTIFY_SOURCE=3 not supported" >&2;} +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -D_FORTIFY_SOURCE=3 not supported" >&5 +printf "%s\n" "$as_me: WARNING: -D_FORTIFY_SOURCE=3 not supported" >&2;} ;; +esac fi fi @@ -9878,8 +10067,8 @@ printf %s "checking if we can add -Wextra... " >&6; } if test ${ac_cv_enable_extra_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS as_fn_append CFLAGS " -Wextra -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -9896,12 +10085,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_enable_extra_warning=yes -else $as_nop - ac_cv_enable_extra_warning=no +else case e in #( + e) ac_cv_enable_extra_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_extra_warning" >&5 printf "%s\n" "$ac_cv_enable_extra_warning" >&6; } @@ -9924,8 +10115,8 @@ printf %s "checking whether $CC accepts and needs -fno-strict-aliasing... " >&6; if test ${ac_cv_no_strict_aliasing+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9961,19 +10152,22 @@ then : ac_cv_no_strict_aliasing=no -else $as_nop - +else case e in #( + e) ac_cv_no_strict_aliasing=yes - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext -else $as_nop - +else case e in #( + e) ac_cv_no_strict_aliasing=no - + ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_no_strict_aliasing" >&5 printf "%s\n" "$ac_cv_no_strict_aliasing" >&6; } @@ -9996,8 +10190,8 @@ printf %s "checking if we can disable $CC unused-result warning... " >&6; } if test ${ac_cv_disable_unused_result_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS as_fn_append CFLAGS " -Wunused-result -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -10014,12 +10208,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_disable_unused_result_warning=yes -else $as_nop - ac_cv_disable_unused_result_warning=no +else case e in #( + e) ac_cv_disable_unused_result_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_unused_result_warning" >&5 printf "%s\n" "$ac_cv_disable_unused_result_warning" >&6; } @@ -10041,8 +10237,8 @@ printf %s "checking if we can disable $CC unused-parameter warning... " >&6; } if test ${ac_cv_disable_unused_parameter_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS as_fn_append CFLAGS " -Wunused-parameter -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -10059,12 +10255,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_disable_unused_parameter_warning=yes -else $as_nop - ac_cv_disable_unused_parameter_warning=no +else case e in #( + e) ac_cv_disable_unused_parameter_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_unused_parameter_warning" >&5 printf "%s\n" "$ac_cv_disable_unused_parameter_warning" >&6; } @@ -10082,8 +10280,8 @@ printf %s "checking if we can disable $CC int-conversion warning... " >&6; } if test ${ac_cv_disable_int_conversion_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS as_fn_append CFLAGS " -Wint-conversion -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -10100,12 +10298,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_disable_int_conversion_warning=yes -else $as_nop - ac_cv_disable_int_conversion_warning=no +else case e in #( + e) ac_cv_disable_int_conversion_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_int_conversion_warning" >&5 printf "%s\n" "$ac_cv_disable_int_conversion_warning" >&6; } @@ -10123,8 +10323,8 @@ printf %s "checking if we can disable $CC missing-field-initializers warning... if test ${ac_cv_disable_missing_field_initializers_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS as_fn_append CFLAGS " -Wmissing-field-initializers -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -10141,12 +10341,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_disable_missing_field_initializers_warning=yes -else $as_nop - ac_cv_disable_missing_field_initializers_warning=no +else case e in #( + e) ac_cv_disable_missing_field_initializers_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_missing_field_initializers_warning" >&5 printf "%s\n" "$ac_cv_disable_missing_field_initializers_warning" >&6; } @@ -10164,8 +10366,8 @@ printf %s "checking if we can enable $CC sign-compare warning... " >&6; } if test ${ac_cv_enable_sign_compare_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS as_fn_append CFLAGS " -Wsign-compare -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -10182,12 +10384,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_enable_sign_compare_warning=yes -else $as_nop - ac_cv_enable_sign_compare_warning=no +else case e in #( + e) ac_cv_enable_sign_compare_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_sign_compare_warning" >&5 printf "%s\n" "$ac_cv_enable_sign_compare_warning" >&6; } @@ -10205,8 +10409,8 @@ printf %s "checking if we can enable $CC unreachable-code warning... " >&6; } if test ${ac_cv_enable_unreachable_code_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS as_fn_append CFLAGS " -Wunreachable-code -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -10223,12 +10427,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_enable_unreachable_code_warning=yes -else $as_nop - ac_cv_enable_unreachable_code_warning=no +else case e in #( + e) ac_cv_enable_unreachable_code_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_unreachable_code_warning" >&5 printf "%s\n" "$ac_cv_enable_unreachable_code_warning" >&6; } @@ -10257,8 +10463,8 @@ printf %s "checking if we can enable $CC strict-prototypes warning... " >&6; } if test ${ac_cv_enable_strict_prototypes_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS as_fn_append CFLAGS " -Wstrict-prototypes -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -10275,12 +10481,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_enable_strict_prototypes_warning=yes -else $as_nop - ac_cv_enable_strict_prototypes_warning=no +else case e in #( + e) ac_cv_enable_strict_prototypes_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_strict_prototypes_warning" >&5 printf "%s\n" "$ac_cv_enable_strict_prototypes_warning" >&6; } @@ -10298,8 +10506,8 @@ printf %s "checking if we can make implicit function declaration an error in $CC if test ${ac_cv_enable_implicit_function_declaration_error+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -10317,12 +10525,14 @@ then : ac_cv_enable_implicit_function_declaration_error=yes -else $as_nop - +else case e in #( + e) ac_cv_enable_implicit_function_declaration_error=no - + ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_implicit_function_declaration_error" >&5 printf "%s\n" "$ac_cv_enable_implicit_function_declaration_error" >&6; } @@ -10340,8 +10550,8 @@ printf %s "checking if we can use visibility in $CC... " >&6; } if test ${ac_cv_enable_visibility+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -10359,12 +10569,14 @@ then : ac_cv_enable_visibility=yes -else $as_nop - +else case e in #( + e) ac_cv_enable_visibility=no - + ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_visibility" >&5 printf "%s\n" "$ac_cv_enable_visibility" >&6; } @@ -10413,8 +10625,8 @@ printf %s "checking whether C compiler accepts -Wunguarded-availability... " >&6 if test ${ax_cv_check_cflags__Werror__Wunguarded_availability+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -Wunguarded-availability" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -10431,19 +10643,22 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__Wunguarded_availability=yes -else $as_nop - ax_cv_check_cflags__Werror__Wunguarded_availability=no +else case e in #( + e) ax_cv_check_cflags__Werror__Wunguarded_availability=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wunguarded_availability" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__Wunguarded_availability" >&6; } if test "x$ax_cv_check_cflags__Werror__Wunguarded_availability" = xyes then : as_fn_append CFLAGS_NODIST " -Werror=unguarded-availability" -else $as_nop - : +else case e in #( + e) : ;; +esac fi @@ -10579,11 +10794,12 @@ if ac_fn_c_try_link "$LINENO" then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } as_fn_error $? "check config.log and use the '--with-universal-archs' option" "$LINENO" 5 - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -10592,8 +10808,8 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac -else $as_nop - +else case e in #( + e) case $ac_sys_system in OpenUNIX*|UnixWare*) BASECFLAGS="$BASECFLAGS -K pentium,host,inline,loop_unroll,alloca " @@ -10602,7 +10818,8 @@ else $as_nop BASECFLAGS="$BASECFLAGS -belf -Ki486 -DSCO5" ;; esac - + ;; +esac fi case "$ac_cv_cc_name" in @@ -10639,12 +10856,12 @@ printf %s "checking whether pthreads are available without options... " >&6; } if test ${ac_cv_pthread_is_default+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_pthread_is_default=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -10668,14 +10885,17 @@ then : ac_cv_kthread=no ac_cv_pthread=no -else $as_nop - ac_cv_pthread_is_default=no +else case e in #( + e) ac_cv_pthread_is_default=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_pthread_is_default" >&5 printf "%s\n" "$ac_cv_pthread_is_default" >&6; } @@ -10695,14 +10915,14 @@ printf %s "checking whether $CC accepts -Kpthread... " >&6; } if test ${ac_cv_kpthread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_cc="$CC" +else case e in #( + e) ac_save_cc="$CC" CC="$CC -Kpthread" if test "$cross_compiling" = yes then : ac_cv_kpthread=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -10722,14 +10942,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_kpthread=yes -else $as_nop - ac_cv_kpthread=no +else case e in #( + e) ac_cv_kpthread=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi -CC="$ac_save_cc" +CC="$ac_save_cc" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_kpthread" >&5 printf "%s\n" "$ac_cv_kpthread" >&6; } @@ -10747,14 +10970,14 @@ printf %s "checking whether $CC accepts -Kthread... " >&6; } if test ${ac_cv_kthread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_cc="$CC" +else case e in #( + e) ac_save_cc="$CC" CC="$CC -Kthread" if test "$cross_compiling" = yes then : ac_cv_kthread=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -10774,14 +10997,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_kthread=yes -else $as_nop - ac_cv_kthread=no +else case e in #( + e) ac_cv_kthread=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi -CC="$ac_save_cc" +CC="$ac_save_cc" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_kthread" >&5 printf "%s\n" "$ac_cv_kthread" >&6; } @@ -10799,14 +11025,14 @@ printf %s "checking whether $CC accepts -pthread... " >&6; } if test ${ac_cv_pthread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_cc="$CC" +else case e in #( + e) ac_save_cc="$CC" CC="$CC -pthread" if test "$cross_compiling" = yes then : ac_cv_pthread=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -10826,14 +11052,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_pthread=yes -else $as_nop - ac_cv_pthread=no +else case e in #( + e) ac_cv_pthread=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi -CC="$ac_save_cc" +CC="$ac_save_cc" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_pthread" >&5 printf "%s\n" "$ac_cv_pthread" >&6; } @@ -10848,8 +11077,8 @@ printf %s "checking whether $CXX also accepts flags for thread support... " >&6; if test ${ac_cv_cxx_thread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_cxx="$CXX" +else case e in #( + e) ac_save_cxx="$CXX" if test "$ac_cv_kpthread" = "yes" then @@ -10880,7 +11109,8 @@ then fi rm -fr conftest* fi -CXX="$ac_save_cxx" +CXX="$ac_save_cxx" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_thread" >&5 printf "%s\n" "$ac_cv_cxx_thread" >&6; } @@ -11407,14 +11637,14 @@ fi ac_header_dirent=no for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do - as_ac_Header=`printf "%s\n" "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh` + as_ac_Header=`printf "%s\n" "ac_cv_header_dirent_$ac_hdr" | sed "$as_sed_sh"` { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_hdr that defines DIR" >&5 printf %s "checking for $ac_hdr that defines DIR... " >&6; } if eval test \${$as_ac_Header+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include <$ac_hdr> @@ -11431,10 +11661,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : eval "$as_ac_Header=yes" -else $as_nop - eval "$as_ac_Header=no" +else case e in #( + e) eval "$as_ac_Header=no" ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi eval ac_res=\$$as_ac_Header { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -11442,7 +11674,7 @@ printf "%s\n" "$ac_res" >&6; } if eval test \"x\$"$as_ac_Header"\" = x"yes" then : cat >>confdefs.h <<_ACEOF -#define `printf "%s\n" "HAVE_$ac_hdr" | $as_tr_cpp` 1 +#define `printf "%s\n" "HAVE_$ac_hdr" | sed "$as_sed_cpp"` 1 _ACEOF ac_header_dirent=$ac_hdr; break @@ -11456,15 +11688,21 @@ printf %s "checking for library containing opendir... " >&6; } if test ${ac_cv_search_opendir+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char opendir (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char opendir (void); int main (void) { @@ -11495,11 +11733,13 @@ done if test ${ac_cv_search_opendir+y} then : -else $as_nop - ac_cv_search_opendir=no +else case e in #( + e) ac_cv_search_opendir=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_opendir" >&5 printf "%s\n" "$ac_cv_search_opendir" >&6; } @@ -11516,15 +11756,21 @@ printf %s "checking for library containing opendir... " >&6; } if test ${ac_cv_search_opendir+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char opendir (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char opendir (void); int main (void) { @@ -11555,11 +11801,13 @@ done if test ${ac_cv_search_opendir+y} then : -else $as_nop - ac_cv_search_opendir=no +else case e in #( + e) ac_cv_search_opendir=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_opendir" >&5 printf "%s\n" "$ac_cv_search_opendir" >&6; } @@ -11752,10 +12000,11 @@ then : printf "%s\n" "#define HAVE_CLOCK_T 1" >>confdefs.h -else $as_nop - +else case e in #( + e) printf "%s\n" "#define clock_t long" >>confdefs.h - + ;; +esac fi @@ -11764,8 +12013,8 @@ printf %s "checking for makedev... " >&6; } if test ${ac_cv_func_makedev+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -11790,12 +12039,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_func_makedev=yes -else $as_nop - ac_cv_func_makedev=no +else case e in #( + e) ac_cv_func_makedev=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_makedev" >&5 printf "%s\n" "$ac_cv_func_makedev" >&6; } @@ -11815,8 +12066,8 @@ printf %s "checking for le64toh... " >&6; } if test ${ac_cv_func_le64toh+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -11839,12 +12090,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_func_le64toh=yes -else $as_nop - ac_cv_func_le64toh=no +else case e in #( + e) ac_cv_func_le64toh=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_le64toh" >&5 printf "%s\n" "$ac_cv_func_le64toh" >&6; } @@ -11894,20 +12147,22 @@ ac_fn_c_check_type "$LINENO" "mode_t" "ac_cv_type_mode_t" "$ac_includes_default" if test "x$ac_cv_type_mode_t" = xyes then : -else $as_nop - +else case e in #( + e) printf "%s\n" "#define mode_t int" >>confdefs.h - + ;; +esac fi ac_fn_c_check_type "$LINENO" "off_t" "ac_cv_type_off_t" "$ac_includes_default" if test "x$ac_cv_type_off_t" = xyes then : -else $as_nop - +else case e in #( + e) printf "%s\n" "#define off_t long int" >>confdefs.h - + ;; +esac fi @@ -11916,8 +12171,8 @@ fi if test "x$ac_cv_type_pid_t" = xyes then : -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #if defined _WIN64 && !defined __CYGWIN__ @@ -11936,14 +12191,16 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_pid_type='int' -else $as_nop - ac_pid_type='__int64' +else case e in #( + e) ac_pid_type='__int64' ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext printf "%s\n" "#define pid_t $ac_pid_type" >>confdefs.h - + ;; +esac fi @@ -11954,42 +12211,33 @@ ac_fn_c_check_type "$LINENO" "size_t" "ac_cv_type_size_t" "$ac_includes_default" if test "x$ac_cv_type_size_t" = xyes then : -else $as_nop - +else case e in #( + e) printf "%s\n" "#define size_t unsigned int" >>confdefs.h - + ;; +esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uid_t in sys/types.h" >&5 -printf %s "checking for uid_t in sys/types.h... " >&6; } -if test ${ac_cv_type_uid_t+y} +ac_fn_c_check_type "$LINENO" "uid_t" "ac_cv_type_uid_t" "$ac_includes_default" +if test "x$ac_cv_type_uid_t" = xyes then : - printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "uid_t" >/dev/null 2>&1 -then : - ac_cv_type_uid_t=yes -else $as_nop - ac_cv_type_uid_t=no +else case e in #( + e) +printf "%s\n" "#define uid_t int" >>confdefs.h + ;; +esac fi -rm -rf conftest* - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_type_uid_t" >&5 -printf "%s\n" "$ac_cv_type_uid_t" >&6; } -if test $ac_cv_type_uid_t = no; then - -printf "%s\n" "#define uid_t int" >>confdefs.h +ac_fn_c_check_type "$LINENO" "gid_t" "ac_cv_type_gid_t" "$ac_includes_default" +if test "x$ac_cv_type_gid_t" = xyes +then : +else case e in #( + e) printf "%s\n" "#define gid_t int" >>confdefs.h - + ;; +esac fi @@ -12018,28 +12266,30 @@ fi # ANSI C requires sizeof(char) == 1, so no need to check it # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of int" >&5 printf %s "checking size of int... " >&6; } if test ${ac_cv_sizeof_int+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (int))" "ac_cv_sizeof_int" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (int))" "ac_cv_sizeof_int" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_int" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_int" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (int) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_int=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_int" >&5 printf "%s\n" "$ac_cv_sizeof_int" >&6; } @@ -12051,28 +12301,30 @@ printf "%s\n" "#define SIZEOF_INT $ac_cv_sizeof_int" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of long" >&5 printf %s "checking size of long... " >&6; } if test ${ac_cv_sizeof_long+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long))" "ac_cv_sizeof_long" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long))" "ac_cv_sizeof_long" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_long" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_long" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (long) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_long=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_long" >&5 printf "%s\n" "$ac_cv_sizeof_long" >&6; } @@ -12089,22 +12341,24 @@ printf %s "checking alignment of long... " >&6; } if test ${ac_cv_alignof_long+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_long" "$ac_includes_default +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_long" "$ac_includes_default typedef struct { char x; long y; } ac__type_alignof_;" then : -else $as_nop - if test "$ac_cv_type_long" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_long" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute alignment of long -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_alignof_long=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_alignof_long" >&5 printf "%s\n" "$ac_cv_alignof_long" >&6; } @@ -12116,28 +12370,30 @@ printf "%s\n" "#define ALIGNOF_LONG $ac_cv_alignof_long" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of long long" >&5 printf %s "checking size of long long... " >&6; } if test ${ac_cv_sizeof_long_long+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long long))" "ac_cv_sizeof_long_long" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long long))" "ac_cv_sizeof_long_long" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_long_long" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_long_long" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (long long) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_long_long=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_long_long" >&5 printf "%s\n" "$ac_cv_sizeof_long_long" >&6; } @@ -12149,28 +12405,30 @@ printf "%s\n" "#define SIZEOF_LONG_LONG $ac_cv_sizeof_long_long" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of void *" >&5 printf %s "checking size of void *... " >&6; } if test ${ac_cv_sizeof_void_p+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (void *))" "ac_cv_sizeof_void_p" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (void *))" "ac_cv_sizeof_void_p" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_void_p" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_void_p" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (void *) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_void_p=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_void_p" >&5 printf "%s\n" "$ac_cv_sizeof_void_p" >&6; } @@ -12182,28 +12440,30 @@ printf "%s\n" "#define SIZEOF_VOID_P $ac_cv_sizeof_void_p" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of short" >&5 printf %s "checking size of short... " >&6; } if test ${ac_cv_sizeof_short+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (short))" "ac_cv_sizeof_short" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (short))" "ac_cv_sizeof_short" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_short" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_short" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (short) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_short=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_short" >&5 printf "%s\n" "$ac_cv_sizeof_short" >&6; } @@ -12215,28 +12475,30 @@ printf "%s\n" "#define SIZEOF_SHORT $ac_cv_sizeof_short" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of float" >&5 printf %s "checking size of float... " >&6; } if test ${ac_cv_sizeof_float+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (float))" "ac_cv_sizeof_float" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (float))" "ac_cv_sizeof_float" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_float" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_float" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (float) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_float=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_float" >&5 printf "%s\n" "$ac_cv_sizeof_float" >&6; } @@ -12248,28 +12510,30 @@ printf "%s\n" "#define SIZEOF_FLOAT $ac_cv_sizeof_float" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of double" >&5 printf %s "checking size of double... " >&6; } if test ${ac_cv_sizeof_double+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (double))" "ac_cv_sizeof_double" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (double))" "ac_cv_sizeof_double" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_double" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_double" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (double) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_double=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_double" >&5 printf "%s\n" "$ac_cv_sizeof_double" >&6; } @@ -12281,28 +12545,30 @@ printf "%s\n" "#define SIZEOF_DOUBLE $ac_cv_sizeof_double" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of fpos_t" >&5 printf %s "checking size of fpos_t... " >&6; } if test ${ac_cv_sizeof_fpos_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (fpos_t))" "ac_cv_sizeof_fpos_t" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (fpos_t))" "ac_cv_sizeof_fpos_t" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_fpos_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_fpos_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (fpos_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_fpos_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_fpos_t" >&5 printf "%s\n" "$ac_cv_sizeof_fpos_t" >&6; } @@ -12314,28 +12580,30 @@ printf "%s\n" "#define SIZEOF_FPOS_T $ac_cv_sizeof_fpos_t" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of size_t" >&5 printf %s "checking size of size_t... " >&6; } if test ${ac_cv_sizeof_size_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (size_t))" "ac_cv_sizeof_size_t" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (size_t))" "ac_cv_sizeof_size_t" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_size_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_size_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (size_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_size_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_size_t" >&5 printf "%s\n" "$ac_cv_sizeof_size_t" >&6; } @@ -12352,22 +12620,24 @@ printf %s "checking alignment of size_t... " >&6; } if test ${ac_cv_alignof_size_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_size_t" "$ac_includes_default +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_size_t" "$ac_includes_default typedef struct { char x; size_t y; } ac__type_alignof_;" then : -else $as_nop - if test "$ac_cv_type_size_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_size_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute alignment of size_t -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_alignof_size_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_alignof_size_t" >&5 printf "%s\n" "$ac_cv_alignof_size_t" >&6; } @@ -12379,28 +12649,30 @@ printf "%s\n" "#define ALIGNOF_SIZE_T $ac_cv_alignof_size_t" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of pid_t" >&5 printf %s "checking size of pid_t... " >&6; } if test ${ac_cv_sizeof_pid_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pid_t))" "ac_cv_sizeof_pid_t" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pid_t))" "ac_cv_sizeof_pid_t" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_pid_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_pid_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (pid_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_pid_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_pid_t" >&5 printf "%s\n" "$ac_cv_sizeof_pid_t" >&6; } @@ -12412,28 +12684,30 @@ printf "%s\n" "#define SIZEOF_PID_T $ac_cv_sizeof_pid_t" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of uintptr_t" >&5 printf %s "checking size of uintptr_t... " >&6; } if test ${ac_cv_sizeof_uintptr_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (uintptr_t))" "ac_cv_sizeof_uintptr_t" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (uintptr_t))" "ac_cv_sizeof_uintptr_t" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_uintptr_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_uintptr_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (uintptr_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_uintptr_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_uintptr_t" >&5 printf "%s\n" "$ac_cv_sizeof_uintptr_t" >&6; } @@ -12450,22 +12724,24 @@ printf %s "checking alignment of max_align_t... " >&6; } if test ${ac_cv_alignof_max_align_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_max_align_t" "$ac_includes_default +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_max_align_t" "$ac_includes_default typedef struct { char x; max_align_t y; } ac__type_alignof_;" then : -else $as_nop - if test "$ac_cv_type_max_align_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_max_align_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute alignment of max_align_t -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_alignof_max_align_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_alignof_max_align_t" >&5 printf "%s\n" "$ac_cv_alignof_max_align_t" >&6; } @@ -12482,8 +12758,8 @@ printf %s "checking for long double... " >&6; } if test ${ac_cv_type_long_double+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$GCC" = yes; then +else case e in #( + e) if test "$GCC" = yes; then ac_cv_type_long_double=yes else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -12506,11 +12782,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_type_long_double=yes -else $as_nop - ac_cv_type_long_double=no +else case e in #( + e) ac_cv_type_long_double=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - fi + fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_type_long_double" >&5 printf "%s\n" "$ac_cv_type_long_double" >&6; } @@ -12522,28 +12800,30 @@ printf "%s\n" "#define HAVE_LONG_DOUBLE 1" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of long double" >&5 printf %s "checking size of long double... " >&6; } if test ${ac_cv_sizeof_long_double+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long double))" "ac_cv_sizeof_long_double" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long double))" "ac_cv_sizeof_long_double" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_long_double" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_long_double" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (long double) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_long_double=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_long_double" >&5 printf "%s\n" "$ac_cv_sizeof_long_double" >&6; } @@ -12556,28 +12836,30 @@ printf "%s\n" "#define SIZEOF_LONG_DOUBLE $ac_cv_sizeof_long_double" >>confdefs. # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of _Bool" >&5 printf %s "checking size of _Bool... " >&6; } if test ${ac_cv_sizeof__Bool+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (_Bool))" "ac_cv_sizeof__Bool" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (_Bool))" "ac_cv_sizeof__Bool" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type__Bool" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type__Bool" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (_Bool) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof__Bool=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof__Bool" >&5 printf "%s\n" "$ac_cv_sizeof__Bool" >&6; } @@ -12590,15 +12872,15 @@ printf "%s\n" "#define SIZEOF__BOOL $ac_cv_sizeof__Bool" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of off_t" >&5 printf %s "checking size of off_t... " >&6; } if test ${ac_cv_sizeof_off_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (off_t))" "ac_cv_sizeof_off_t" " +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (off_t))" "ac_cv_sizeof_off_t" " #ifdef HAVE_SYS_TYPES_H #include #endif @@ -12606,17 +12888,19 @@ else $as_nop " then : -else $as_nop - if test "$ac_cv_type_off_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_off_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (off_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_off_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_off_t" >&5 printf "%s\n" "$ac_cv_sizeof_off_t" >&6; } @@ -12651,24 +12935,25 @@ printf "%s\n" "#define HAVE_LARGEFILE_SUPPORT 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - + ;; +esac fi # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of time_t" >&5 printf %s "checking size of time_t... " >&6; } if test ${ac_cv_sizeof_time_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (time_t))" "ac_cv_sizeof_time_t" " +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (time_t))" "ac_cv_sizeof_time_t" " #ifdef HAVE_SYS_TYPES_H #include #endif @@ -12679,17 +12964,19 @@ else $as_nop " then : -else $as_nop - if test "$ac_cv_type_time_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_time_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (time_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_time_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_time_t" >&5 printf "%s\n" "$ac_cv_sizeof_time_t" >&6; } @@ -12715,8 +13002,8 @@ printf %s "checking for pthread_t... " >&6; } if test ${ac_cv_have_pthread_t+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -12733,11 +13020,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_have_pthread_t=yes -else $as_nop - ac_cv_have_pthread_t=no +else case e in #( + e) ac_cv_have_pthread_t=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_pthread_t" >&5 printf "%s\n" "$ac_cv_have_pthread_t" >&6; } @@ -12746,15 +13035,15 @@ then : # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of pthread_t" >&5 printf %s "checking size of pthread_t... " >&6; } if test ${ac_cv_sizeof_pthread_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_t))" "ac_cv_sizeof_pthread_t" " +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_t))" "ac_cv_sizeof_pthread_t" " #ifdef HAVE_PTHREAD_H #include #endif @@ -12762,17 +13051,19 @@ else $as_nop " then : -else $as_nop - if test "$ac_cv_type_pthread_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_pthread_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (pthread_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_pthread_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_pthread_t" >&5 printf "%s\n" "$ac_cv_sizeof_pthread_t" >&6; } @@ -12789,29 +13080,31 @@ fi # This checking will be unnecessary after removing deprecated TLS API. # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of pthread_key_t" >&5 printf %s "checking size of pthread_key_t... " >&6; } if test ${ac_cv_sizeof_pthread_key_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_key_t))" "ac_cv_sizeof_pthread_key_t" "#include +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_key_t))" "ac_cv_sizeof_pthread_key_t" "#include " then : -else $as_nop - if test "$ac_cv_type_pthread_key_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_pthread_key_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (pthread_key_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_pthread_key_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_pthread_key_t" >&5 printf "%s\n" "$ac_cv_sizeof_pthread_key_t" >&6; } @@ -12826,8 +13119,8 @@ printf %s "checking whether pthread_key_t is compatible with int... " >&6; } if test ${ac_cv_pthread_key_t_is_arithmetic_type+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$ac_cv_sizeof_pthread_key_t" -eq "$ac_cv_sizeof_int" ; then cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -12843,15 +13136,17 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_pthread_key_t_is_arithmetic_type=yes -else $as_nop - ac_cv_pthread_key_t_is_arithmetic_type=no - +else case e in #( + e) ac_cv_pthread_key_t_is_arithmetic_type=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext else ac_cv_pthread_key_t_is_arithmetic_type=no fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_pthread_key_t_is_arithmetic_type" >&5 printf "%s\n" "$ac_cv_pthread_key_t_is_arithmetic_type" >&6; } @@ -12910,9 +13205,10 @@ printf "%s\n" "yes" >&6; }; else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; }; DSYMUTIL= fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -12924,8 +13220,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_DSYMUTIL_PATH+y} then : printf %s "(cached) " >&6 -else $as_nop - case $DSYMUTIL_PATH in +else case e in #( + e) case $DSYMUTIL_PATH in [\\/]* | ?:[\\/]*) ac_cv_path_DSYMUTIL_PATH="$DSYMUTIL_PATH" # Let the user override the test with a path. ;; @@ -12951,6 +13247,7 @@ IFS=$as_save_IFS test -z "$ac_cv_path_DSYMUTIL_PATH" && ac_cv_path_DSYMUTIL_PATH="not found" ;; +esac ;; esac fi DSYMUTIL_PATH=$ac_cv_path_DSYMUTIL_PATH @@ -12998,9 +13295,10 @@ LDFLAGS="-fsanitize=address $LDFLAGS" # ASan works by controlling memory allocation, our own malloc interferes. with_pymalloc="no" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -13018,8 +13316,8 @@ printf %s "checking whether C compiler accepts -fsanitize=memory... " >&6; } if test ${ax_cv_check_cflags___fsanitize_memory+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -fsanitize=memory" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -13036,11 +13334,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags___fsanitize_memory=yes -else $as_nop - ax_cv_check_cflags___fsanitize_memory=no +else case e in #( + e) ax_cv_check_cflags___fsanitize_memory=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___fsanitize_memory" >&5 printf "%s\n" "$ax_cv_check_cflags___fsanitize_memory" >&6; } @@ -13050,16 +13350,18 @@ then : BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" -else $as_nop - as_fn_error $? "The selected compiler doesn't support memory sanitizer" "$LINENO" 5 +else case e in #( + e) as_fn_error $? "The selected compiler doesn't support memory sanitizer" "$LINENO" 5 ;; +esac fi # MSan works by controlling memory allocation, our own malloc interferes. with_pymalloc="no" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -13076,12 +13378,13 @@ BASECFLAGS="-fsanitize=undefined $BASECFLAGS" LDFLAGS="-fsanitize=undefined $LDFLAGS" with_ubsan="yes" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } with_ubsan="no" - + ;; +esac fi @@ -13098,12 +13401,13 @@ BASECFLAGS="-fsanitize=thread $BASECFLAGS" LDFLAGS="-fsanitize=thread $LDFLAGS" with_tsan="yes" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } with_tsan="no" - + ;; +esac fi @@ -13487,16 +13791,22 @@ printf %s "checking for sendfile in -lsendfile... " >&6; } if test ${ac_cv_lib_sendfile_sendfile+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsendfile $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sendfile (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sendfile (void); int main (void) { @@ -13508,12 +13818,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sendfile_sendfile=yes -else $as_nop - ac_cv_lib_sendfile_sendfile=no +else case e in #( + e) ac_cv_lib_sendfile_sendfile=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sendfile_sendfile" >&5 printf "%s\n" "$ac_cv_lib_sendfile_sendfile" >&6; } @@ -13530,16 +13842,22 @@ printf %s "checking for dlopen in -ldl... " >&6; } if test ${ac_cv_lib_dl_dlopen+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-ldl $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char dlopen (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char dlopen (void); int main (void) { @@ -13551,12 +13869,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_dl_dlopen=yes -else $as_nop - ac_cv_lib_dl_dlopen=no +else case e in #( + e) ac_cv_lib_dl_dlopen=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5 printf "%s\n" "$ac_cv_lib_dl_dlopen" >&6; } @@ -13573,16 +13893,22 @@ printf %s "checking for shl_load in -ldld... " >&6; } if test ${ac_cv_lib_dld_shl_load+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-ldld $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char shl_load (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char shl_load (void); int main (void) { @@ -13594,12 +13920,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_dld_shl_load=yes -else $as_nop - ac_cv_lib_dld_shl_load=no +else case e in #( + e) ac_cv_lib_dld_shl_load=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dld_shl_load" >&5 printf "%s\n" "$ac_cv_lib_dld_shl_load" >&6; } @@ -13627,12 +13955,12 @@ then : for ac_func in uuid_create uuid_enc_be do : - as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | $as_tr_sh` + as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | sed "$as_sed_sh"` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" if eval test \"x\$"$as_ac_var"\" = x"yes" then : cat >>confdefs.h <<_ACEOF -#define `printf "%s\n" "HAVE_$ac_func" | $as_tr_cpp` 1 +#define `printf "%s\n" "HAVE_$ac_func" | sed "$as_sed_cpp"` 1 _ACEOF have_uuid=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} @@ -13730,16 +14058,22 @@ printf %s "checking for uuid_generate_time in -luuid... " >&6; } if test ${ac_cv_lib_uuid_uuid_generate_time+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-luuid $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char uuid_generate_time (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char uuid_generate_time (void); int main (void) { @@ -13751,12 +14085,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_uuid_uuid_generate_time=yes -else $as_nop - ac_cv_lib_uuid_uuid_generate_time=no +else case e in #( + e) ac_cv_lib_uuid_uuid_generate_time=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_uuid_uuid_generate_time" >&5 printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time" >&6; } @@ -13773,16 +14109,22 @@ printf %s "checking for uuid_generate_time_safe in -luuid... " >&6; } if test ${ac_cv_lib_uuid_uuid_generate_time_safe+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-luuid $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char uuid_generate_time_safe (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char uuid_generate_time_safe (void); int main (void) { @@ -13794,12 +14136,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_uuid_uuid_generate_time_safe=yes -else $as_nop - ac_cv_lib_uuid_uuid_generate_time_safe=no +else case e in #( + e) ac_cv_lib_uuid_uuid_generate_time_safe=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_uuid_uuid_generate_time_safe" >&5 printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time_safe" >&6; } @@ -13856,16 +14200,22 @@ printf %s "checking for uuid_generate_time in -luuid... " >&6; } if test ${ac_cv_lib_uuid_uuid_generate_time+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-luuid $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char uuid_generate_time (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char uuid_generate_time (void); int main (void) { @@ -13877,12 +14227,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_uuid_uuid_generate_time=yes -else $as_nop - ac_cv_lib_uuid_uuid_generate_time=no +else case e in #( + e) ac_cv_lib_uuid_uuid_generate_time=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_uuid_uuid_generate_time" >&5 printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time" >&6; } @@ -13899,16 +14251,22 @@ printf %s "checking for uuid_generate_time_safe in -luuid... " >&6; } if test ${ac_cv_lib_uuid_uuid_generate_time_safe+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-luuid $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char uuid_generate_time_safe (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char uuid_generate_time_safe (void); int main (void) { @@ -13920,12 +14278,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_uuid_uuid_generate_time_safe=yes -else $as_nop - ac_cv_lib_uuid_uuid_generate_time_safe=no +else case e in #( + e) ac_cv_lib_uuid_uuid_generate_time_safe=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_uuid_uuid_generate_time_safe" >&5 printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time_safe" >&6; } @@ -14020,15 +14380,21 @@ printf %s "checking for library containing sem_init... " >&6; } if test ${ac_cv_search_sem_init+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sem_init (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sem_init (void); int main (void) { @@ -14059,11 +14425,13 @@ done if test ${ac_cv_search_sem_init+y} then : -else $as_nop - ac_cv_search_sem_init=no +else case e in #( + e) ac_cv_search_sem_init=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_sem_init" >&5 printf "%s\n" "$ac_cv_search_sem_init" >&6; } @@ -14081,16 +14449,22 @@ printf %s "checking for textdomain in -lintl... " >&6; } if test ${ac_cv_lib_intl_textdomain+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lintl $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char textdomain (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char textdomain (void); int main (void) { @@ -14102,12 +14476,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_intl_textdomain=yes -else $as_nop - ac_cv_lib_intl_textdomain=no +else case e in #( + e) ac_cv_lib_intl_textdomain=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_intl_textdomain" >&5 printf "%s\n" "$ac_cv_lib_intl_textdomain" >&6; } @@ -14146,11 +14522,12 @@ printf "%s\n" "#define AIX_GENUINE_CPLUSPLUS 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -14179,8 +14556,8 @@ esac if test "$cross_compiling" = yes then : ac_cv_c_complex_supported=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -14201,11 +14578,13 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_c_complex_supported=yes -else $as_nop - ac_cv_c_complex_supported=no +else case e in #( + e) ac_cv_c_complex_supported=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi if test "$ac_cv_c_complex_supported" = "yes"; then @@ -14220,8 +14599,8 @@ printf %s "checking aligned memory access is required... " >&6; } if test ${ac_cv_aligned_required+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : # "yes" changes the hash function to FNV, which causes problems with Numba @@ -14231,8 +14610,8 @@ if test "$ac_sys_system" = "Linux-android"; then else ac_cv_aligned_required=yes fi -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main(void) @@ -14251,14 +14630,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_aligned_required=no -else $as_nop - ac_cv_aligned_required=yes +else case e in #( + e) ac_cv_aligned_required=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_aligned_required" >&5 printf "%s\n" "$ac_cv_aligned_required" >&6; } @@ -14298,9 +14680,10 @@ case "$withval" in ;; esac -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default" >&5 -printf "%s\n" "default" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default" >&5 +printf "%s\n" "default" >&6; } ;; +esac fi @@ -14338,10 +14721,11 @@ printf "%s\n" "\"$withval\"" >&6; } ;; esac -else $as_nop - validate_tzpath "$TZPATH" +else case e in #( + e) validate_tzpath "$TZPATH" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: \"$TZPATH\"" >&5 -printf "%s\n" "\"$TZPATH\"" >&6; } +printf "%s\n" "\"$TZPATH\"" >&6; } ;; +esac fi @@ -14352,16 +14736,22 @@ printf %s "checking for t_open in -lnsl... " >&6; } if test ${ac_cv_lib_nsl_t_open+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lnsl $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char t_open (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char t_open (void); int main (void) { @@ -14373,12 +14763,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_nsl_t_open=yes -else $as_nop - ac_cv_lib_nsl_t_open=no +else case e in #( + e) ac_cv_lib_nsl_t_open=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nsl_t_open" >&5 printf "%s\n" "$ac_cv_lib_nsl_t_open" >&6; } @@ -14392,16 +14784,22 @@ printf %s "checking for socket in -lsocket... " >&6; } if test ${ac_cv_lib_socket_socket+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsocket $LIBS $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char socket (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char socket (void); int main (void) { @@ -14413,12 +14811,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_socket_socket=yes -else $as_nop - ac_cv_lib_socket_socket=no +else case e in #( + e) ac_cv_lib_socket_socket=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_socket_socket" >&5 printf "%s\n" "$ac_cv_lib_socket_socket" >&6; } @@ -14435,16 +14835,22 @@ printf %s "checking for socket in -lnetwork... " >&6; } if test ${ac_cv_lib_network_socket+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lnetwork $LIBS $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char socket (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char socket (void); int main (void) { @@ -14456,12 +14862,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_network_socket=yes -else $as_nop - ac_cv_lib_network_socket=no +else case e in #( + e) ac_cv_lib_network_socket=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_network_socket" >&5 printf "%s\n" "$ac_cv_lib_network_socket" >&6; } @@ -14484,9 +14892,10 @@ then : printf "%s\n" "$withval" >&6; } LIBS="$withval $LIBS" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -14498,8 +14907,9 @@ printf %s "checking for --with-system-expat... " >&6; } if test ${with_system_expat+y} then : withval=$with_system_expat; -else $as_nop - with_system_expat="no" +else case e in #( + e) with_system_expat="no" ;; +esac fi @@ -14513,12 +14923,13 @@ then : LIBEXPAT_LDFLAGS=${LIBEXPAT_LDFLAGS-"-lexpat"} LIBEXPAT_INTERNAL= -else $as_nop - +else case e in #( + e) LIBEXPAT_CFLAGS="-I\$(srcdir)/Modules/expat" LIBEXPAT_LDFLAGS="-lm \$(LIBEXPAT_A)" LIBEXPAT_INTERNAL="\$(LIBEXPAT_HEADERS) \$(LIBEXPAT_A)" - + ;; +esac fi @@ -14544,16 +14955,22 @@ printf %s "checking for ffi_call in -lffi... " >&6; } if test ${ac_cv_lib_ffi_ffi_call+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lffi $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char ffi_call (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ffi_call (void); int main (void) { @@ -14565,12 +14982,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_ffi_ffi_call=yes -else $as_nop - ac_cv_lib_ffi_ffi_call=no +else case e in #( + e) ac_cv_lib_ffi_ffi_call=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5 printf "%s\n" "$ac_cv_lib_ffi_ffi_call" >&6; } @@ -14675,17 +15094,23 @@ printf %s "checking for ffi_call in -lffi... " >&6; } if test ${ac_cv_lib_ffi_ffi_call+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lffi $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char ffi_call (); -int + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ffi_call (void); +int main (void) { return ffi_call (); @@ -14696,12 +15121,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_ffi_ffi_call=yes -else $as_nop - ac_cv_lib_ffi_ffi_call=no +else case e in #( + e) ac_cv_lib_ffi_ffi_call=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5 printf "%s\n" "$ac_cv_lib_ffi_ffi_call" >&6; } @@ -14712,8 +15139,9 @@ then : LIBFFI_CFLAGS=${LIBFFI_CFLAGS-""} LIBFFI_LIBS=${LIBFFI_LIBS-"-lffi"} -else $as_nop - have_libffi=no +else case e in #( + e) have_libffi=no ;; +esac fi @@ -14748,16 +15176,22 @@ printf %s "checking for ffi_call in -lffi... " >&6; } if test ${ac_cv_lib_ffi_ffi_call+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lffi $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char ffi_call (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ffi_call (void); int main (void) { @@ -14769,12 +15203,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_ffi_ffi_call=yes -else $as_nop - ac_cv_lib_ffi_ffi_call=no +else case e in #( + e) ac_cv_lib_ffi_ffi_call=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5 printf "%s\n" "$ac_cv_lib_ffi_ffi_call" >&6; } @@ -14785,8 +15221,9 @@ then : LIBFFI_CFLAGS=${LIBFFI_CFLAGS-""} LIBFFI_LIBS=${LIBFFI_LIBS-"-lffi"} -else $as_nop - have_libffi=no +else case e in #( + e) have_libffi=no ;; +esac fi @@ -14859,8 +15296,8 @@ printf %s "checking for ffi_prep_cif_var... " >&6; } if test ${ac_cv_func_ffi_prep_cif_var+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -14874,11 +15311,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_ffi_prep_cif_var=yes -else $as_nop - ac_cv_func_ffi_prep_cif_var=no +else case e in #( + e) ac_cv_func_ffi_prep_cif_var=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_ffi_prep_cif_var" >&5 printf "%s\n" "$ac_cv_func_ffi_prep_cif_var" >&6; } @@ -14898,8 +15337,8 @@ printf %s "checking for ffi_prep_closure_loc... " >&6; } if test ${ac_cv_func_ffi_prep_closure_loc+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -14913,11 +15352,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_ffi_prep_closure_loc=yes -else $as_nop - ac_cv_func_ffi_prep_closure_loc=no +else case e in #( + e) ac_cv_func_ffi_prep_closure_loc=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_ffi_prep_closure_loc" >&5 printf "%s\n" "$ac_cv_func_ffi_prep_closure_loc" >&6; } @@ -14937,8 +15378,8 @@ printf %s "checking for ffi_closure_alloc... " >&6; } if test ${ac_cv_func_ffi_closure_alloc+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -14952,11 +15393,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_ffi_closure_alloc=yes -else $as_nop - ac_cv_func_ffi_closure_alloc=no +else case e in #( + e) ac_cv_func_ffi_closure_alloc=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_ffi_closure_alloc" >&5 printf "%s\n" "$ac_cv_func_ffi_closure_alloc" >&6; } @@ -14989,8 +15432,8 @@ printf %s "checking libffi has complex type support... " >&6; } if test ${ac_cv_ffi_complex_double_supported+y} then : printf %s "(cached) " >&6 -else $as_nop - save_CFLAGS=$CFLAGS +else case e in #( + e) save_CFLAGS=$CFLAGS save_CPPFLAGS=$CPPFLAGS save_LDFLAGS=$LDFLAGS save_LIBS=$LIBS @@ -15002,8 +15445,8 @@ save_LIBS=$LIBS if test "$cross_compiling" = yes then : ac_cv_ffi_complex_double_supported=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -15033,11 +15476,13 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_ffi_complex_double_supported=yes -else $as_nop - ac_cv_ffi_complex_double_supported=no +else case e in #( + e) ac_cv_ffi_complex_double_supported=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi @@ -15046,7 +15491,8 @@ CPPFLAGS=$save_CPPFLAGS LDFLAGS=$save_LDFLAGS LIBS=$save_LIBS - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_ffi_complex_double_supported" >&5 printf "%s\n" "$ac_cv_ffi_complex_double_supported" >&6; } @@ -15064,8 +15510,9 @@ printf %s "checking for --with-system-libmpdec... " >&6; } if test ${with_system_libmpdec+y} then : withval=$with_system_libmpdec; -else $as_nop - with_system_libmpdec="yes" +else case e in #( + e) with_system_libmpdec="yes" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 @@ -15150,12 +15597,13 @@ else printf "%s\n" "yes" >&6; } fi -else $as_nop - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" +else case e in #( + e) LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" have_mpdec=yes - with_system_libmpdec=no + with_system_libmpdec=no ;; +esac fi if test "x$with_system_libmpdec" = xyes @@ -15189,8 +15637,9 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : have_mpdec=yes -else $as_nop - have_mpdec=no +else case e in #( + e) have_mpdec=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -15201,9 +15650,10 @@ LDFLAGS=$save_LDFLAGS LIBS=$save_LIBS -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 -printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 +printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} ;; +esac fi if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" @@ -15231,8 +15681,9 @@ printf %s "checking for --with-decimal-contextvar... " >&6; } if test ${with_decimal_contextvar+y} then : withval=$with_decimal_contextvar; -else $as_nop - with_decimal_contextvar="yes" +else case e in #( + e) with_decimal_contextvar="yes" ;; +esac fi @@ -15470,16 +15921,22 @@ printf %s "checking for sqlite3_bind_double in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_bind_double+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_bind_double (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_bind_double (void); int main (void) { @@ -15491,12 +15948,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_bind_double=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_bind_double=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_bind_double=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_bind_double" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_bind_double" >&6; } @@ -15506,10 +15965,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15519,16 +15979,22 @@ printf %s "checking for sqlite3_column_decltype in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_column_decltype+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_column_decltype (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_column_decltype (void); int main (void) { @@ -15540,12 +16006,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_column_decltype=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_column_decltype=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_column_decltype=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_column_decltype" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_column_decltype" >&6; } @@ -15555,10 +16023,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15568,16 +16037,22 @@ printf %s "checking for sqlite3_column_double in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_column_double+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_column_double (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_column_double (void); int main (void) { @@ -15589,12 +16064,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_column_double=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_column_double=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_column_double=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_column_double" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_column_double" >&6; } @@ -15604,10 +16081,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15617,16 +16095,22 @@ printf %s "checking for sqlite3_complete in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_complete+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_complete (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_complete (void); int main (void) { @@ -15638,12 +16122,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_complete=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_complete=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_complete=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_complete" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_complete" >&6; } @@ -15653,10 +16139,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15666,16 +16153,22 @@ printf %s "checking for sqlite3_progress_handler in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_progress_handler+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_progress_handler (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_progress_handler (void); int main (void) { @@ -15687,12 +16180,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_progress_handler=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_progress_handler=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_progress_handler=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_progress_handler" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_progress_handler" >&6; } @@ -15702,10 +16197,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15715,16 +16211,22 @@ printf %s "checking for sqlite3_result_double in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_result_double+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_result_double (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_result_double (void); int main (void) { @@ -15736,12 +16238,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_result_double=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_result_double=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_result_double=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_result_double" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_result_double" >&6; } @@ -15751,10 +16255,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15764,16 +16269,22 @@ printf %s "checking for sqlite3_set_authorizer in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_set_authorizer+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_set_authorizer (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_set_authorizer (void); int main (void) { @@ -15785,12 +16296,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_set_authorizer=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_set_authorizer=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_set_authorizer=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_set_authorizer" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_set_authorizer" >&6; } @@ -15800,10 +16313,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15813,16 +16327,22 @@ printf %s "checking for sqlite3_trace_v2 in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_trace_v2+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_trace_v2 (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_trace_v2 (void); int main (void) { @@ -15834,12 +16354,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_trace_v2=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_trace_v2=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_trace_v2=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_trace_v2" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_trace_v2" >&6; } @@ -15849,8 +16371,8 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sqlite3_trace in -lsqlite3" >&5 @@ -15858,16 +16380,22 @@ printf %s "checking for sqlite3_trace in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_trace+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_trace (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_trace (void); int main (void) { @@ -15879,12 +16407,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_trace=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_trace=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_trace=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_trace" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_trace" >&6; } @@ -15894,15 +16424,17 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi - + ;; +esac fi @@ -15912,16 +16444,22 @@ printf %s "checking for sqlite3_value_double in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_value_double+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_value_double (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_value_double (void); int main (void) { @@ -15933,12 +16471,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_value_double=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_value_double=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_value_double=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_value_double" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_value_double" >&6; } @@ -15948,10 +16488,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15960,16 +16501,22 @@ printf %s "checking for sqlite3_load_extension in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_load_extension+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_load_extension (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_load_extension (void); int main (void) { @@ -15981,21 +16528,24 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_load_extension=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_load_extension=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_load_extension=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_load_extension" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_load_extension" >&6; } if test "x$ac_cv_lib_sqlite3_sqlite3_load_extension" = xyes then : have_sqlite3_load_extension=yes -else $as_nop - have_sqlite3_load_extension=no - +else case e in #( + e) have_sqlite3_load_extension=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sqlite3_serialize in -lsqlite3" >&5 @@ -16003,16 +16553,22 @@ printf %s "checking for sqlite3_serialize in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_serialize+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_serialize (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_serialize (void); int main (void) { @@ -16024,12 +16580,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_serialize=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_serialize=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_serialize=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_serialize" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_serialize" >&6; } @@ -16043,10 +16601,11 @@ printf "%s\n" "#define PY_SQLITE_HAVE_SERIALIZE 1" >>confdefs.h fi -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext @@ -16074,22 +16633,24 @@ printf "%s\n" "n/a" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: Your version of SQLite does not support loadable extensions" >&5 printf "%s\n" "$as_me: WARNING: Your version of SQLite does not support loadable extensions" >&2;} -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } printf "%s\n" "#define PY_SQLITE_ENABLE_LOAD_EXTENSION 1" >>confdefs.h - + ;; +esac fi -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - + ;; +esac fi @@ -16277,8 +16838,8 @@ See the pkg-config man page for more details." "$LINENO" 5 elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full path to pkg-config. @@ -16288,7 +16849,7 @@ and X11_LIBS to avoid the need to call pkg-config. See the pkg-config man page for more details. To get pkg-config, see . -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else X11_CFLAGS=$pkg_cv_X11_CFLAGS X11_LIBS=$pkg_cv_X11_LIBS @@ -16356,10 +16917,11 @@ then : have_tcltk=yes as_fn_append TCLTK_CFLAGS " -Wno-strict-prototypes -DWITH_APPINIT=1" -else $as_nop - +else case e in #( + e) have_tcltk=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -16393,16 +16955,22 @@ printf %s "checking for gdbm_open in -lgdbm... " >&6; } if test ${ac_cv_lib_gdbm_gdbm_open+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lgdbm $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char gdbm_open (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char gdbm_open (void); int main (void) { @@ -16414,12 +16982,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_gdbm_gdbm_open=yes -else $as_nop - ac_cv_lib_gdbm_gdbm_open=no +else case e in #( + e) ac_cv_lib_gdbm_gdbm_open=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gdbm_gdbm_open" >&5 printf "%s\n" "$ac_cv_lib_gdbm_gdbm_open" >&6; } @@ -16429,13 +16999,15 @@ then : have_gdbm=yes GDBM_LIBS=${GDBM_LIBS-"-lgdbm"} -else $as_nop - have_gdbm=no +else case e in #( + e) have_gdbm=no ;; +esac fi -else $as_nop - have_gdbm=no +else case e in #( + e) have_gdbm=no ;; +esac fi done @@ -16465,15 +17037,21 @@ printf %s "checking for library containing dbm_open... " >&6; } if test ${ac_cv_search_dbm_open+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char dbm_open (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char dbm_open (void); int main (void) { @@ -16504,11 +17082,13 @@ done if test ${ac_cv_search_dbm_open+y} then : -else $as_nop - ac_cv_search_dbm_open=no +else case e in #( + e) ac_cv_search_dbm_open=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_dbm_open" >&5 printf "%s\n" "$ac_cv_search_dbm_open" >&6; } @@ -16557,18 +17137,20 @@ printf "%s\n" "$have_ndbm ($dbm_ndbm)" >&6; } if test ${ac_cv_header_gdbm_slash_ndbm_h+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ac_fn_c_check_header_compile "$LINENO" "gdbm/ndbm.h" "ac_cv_header_gdbm_ndbm_h" "$ac_includes_default" if test "x$ac_cv_header_gdbm_ndbm_h" = xyes then : ac_cv_header_gdbm_slash_ndbm_h=yes -else $as_nop - ac_cv_header_gdbm_slash_ndbm_h=no - +else case e in #( + e) ac_cv_header_gdbm_slash_ndbm_h=no + ;; +esac fi - + ;; +esac fi if test "x$ac_cv_header_gdbm_slash_ndbm_h" = xyes @@ -16584,18 +17166,20 @@ fi if test ${ac_cv_header_gdbm_dash_ndbm_h+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ac_fn_c_check_header_compile "$LINENO" "gdbm-ndbm.h" "ac_cv_header_gdbm_ndbm_h" "$ac_includes_default" if test "x$ac_cv_header_gdbm_ndbm_h" = xyes then : ac_cv_header_gdbm_dash_ndbm_h=yes -else $as_nop - ac_cv_header_gdbm_dash_ndbm_h=no - +else case e in #( + e) ac_cv_header_gdbm_dash_ndbm_h=no + ;; +esac fi - + ;; +esac fi if test "x$ac_cv_header_gdbm_dash_ndbm_h" = xyes @@ -16621,15 +17205,21 @@ printf %s "checking for library containing dbm_open... " >&6; } if test ${ac_cv_search_dbm_open+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char dbm_open (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char dbm_open (void); int main (void) { @@ -16660,11 +17250,13 @@ done if test ${ac_cv_search_dbm_open+y} then : -else $as_nop - ac_cv_search_dbm_open=no +else case e in #( + e) ac_cv_search_dbm_open=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_dbm_open" >&5 printf "%s\n" "$ac_cv_search_dbm_open" >&6; } @@ -16673,8 +17265,9 @@ if test "$ac_res" != no then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" have_gdbm_compat=yes -else $as_nop - have_gdbm_compat=no +else case e in #( + e) have_gdbm_compat=no ;; +esac fi @@ -16700,8 +17293,8 @@ printf %s "checking for libdb... " >&6; } if test ${ac_cv_have_libdb+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) save_CFLAGS=$CFLAGS save_CPPFLAGS=$CPPFLAGS save_LDFLAGS=$LDFLAGS @@ -16730,8 +17323,9 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_have_libdb=yes -else $as_nop - ac_cv_have_libdb=no +else case e in #( + e) ac_cv_have_libdb=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -16742,7 +17336,8 @@ LDFLAGS=$save_LDFLAGS LIBS=$save_LIBS - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_libdb" >&5 printf "%s\n" "$ac_cv_have_libdb" >&6; } @@ -16767,8 +17362,9 @@ printf %s "checking for --with-dbmliborder... " >&6; } if test ${with_dbmliborder+y} then : withval=$with_dbmliborder; -else $as_nop - with_dbmliborder=gdbm:ndbm:bdb +else case e in #( + e) with_dbmliborder=gdbm:ndbm:bdb ;; +esac fi @@ -16885,8 +17481,8 @@ printf %s "checking for _POSIX_THREADS defined in unistd.h... " >&6; } if test ${ac_cv_defined__POSIX_THREADS_unistd_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -16907,18 +17503,21 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined__POSIX_THREADS_unistd_h=yes -else $as_nop - ac_cv_defined__POSIX_THREADS_unistd_h=no +else case e in #( + e) ac_cv_defined__POSIX_THREADS_unistd_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__POSIX_THREADS_unistd_h" >&5 printf "%s\n" "$ac_cv_defined__POSIX_THREADS_unistd_h" >&6; } if test $ac_cv_defined__POSIX_THREADS_unistd_h != "no" then : unistd_defines_pthreads=yes -else $as_nop - unistd_defines_pthreads=no +else case e in #( + e) unistd_defines_pthreads=no ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $unistd_defines_pthreads" >&5 printf "%s\n" "$unistd_defines_pthreads" >&6; } @@ -16956,8 +17555,8 @@ then : printf "%s\n" "yes" >&6; } posix_threads=yes -else $as_nop - +else case e in #( + e) LIBS=$_libs ac_fn_c_check_func "$LINENO" "pthread_detach" "ac_cv_func_pthread_detach" if test "x$ac_cv_func_pthread_detach" = xyes @@ -16965,23 +17564,29 @@ then : posix_threads=yes -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pthread_create in -lpthreads" >&5 printf %s "checking for pthread_create in -lpthreads... " >&6; } if test ${ac_cv_lib_pthreads_pthread_create+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lpthreads $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char pthread_create (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char pthread_create (void); int main (void) { @@ -16993,12 +17598,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_pthreads_pthread_create=yes -else $as_nop - ac_cv_lib_pthreads_pthread_create=no +else case e in #( + e) ac_cv_lib_pthreads_pthread_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_pthreads_pthread_create" >&5 printf "%s\n" "$ac_cv_lib_pthreads_pthread_create" >&6; } @@ -17008,23 +17615,29 @@ then : posix_threads=yes LIBS="$LIBS -lpthreads" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pthread_create in -lc_r" >&5 printf %s "checking for pthread_create in -lc_r... " >&6; } if test ${ac_cv_lib_c_r_pthread_create+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lc_r $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char pthread_create (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char pthread_create (void); int main (void) { @@ -17036,12 +17649,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_c_r_pthread_create=yes -else $as_nop - ac_cv_lib_c_r_pthread_create=no +else case e in #( + e) ac_cv_lib_c_r_pthread_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_c_r_pthread_create" >&5 printf "%s\n" "$ac_cv_lib_c_r_pthread_create" >&6; } @@ -17051,23 +17666,29 @@ then : posix_threads=yes LIBS="$LIBS -lc_r" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __pthread_create_system in -lpthread" >&5 printf %s "checking for __pthread_create_system in -lpthread... " >&6; } if test ${ac_cv_lib_pthread___pthread_create_system+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lpthread $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char __pthread_create_system (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char __pthread_create_system (void); int main (void) { @@ -17079,12 +17700,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_pthread___pthread_create_system=yes -else $as_nop - ac_cv_lib_pthread___pthread_create_system=no +else case e in #( + e) ac_cv_lib_pthread___pthread_create_system=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_pthread___pthread_create_system" >&5 printf "%s\n" "$ac_cv_lib_pthread___pthread_create_system" >&6; } @@ -17094,23 +17717,29 @@ then : posix_threads=yes LIBS="$LIBS -lpthread" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pthread_create in -lcma" >&5 printf %s "checking for pthread_create in -lcma... " >&6; } if test ${ac_cv_lib_cma_pthread_create+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lcma $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char pthread_create (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char pthread_create (void); int main (void) { @@ -17122,12 +17751,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_cma_pthread_create=yes -else $as_nop - ac_cv_lib_cma_pthread_create=no +else case e in #( + e) ac_cv_lib_cma_pthread_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_cma_pthread_create" >&5 printf "%s\n" "$ac_cv_lib_cma_pthread_create" >&6; } @@ -17137,8 +17768,8 @@ then : posix_threads=yes LIBS="$LIBS -lcma" -else $as_nop - +else case e in #( + e) case $ac_sys_system in #( WASI) : posix_threads=stub ;; #( @@ -17146,17 +17777,23 @@ else $as_nop as_fn_error $? "could not find pthreads on your system" "$LINENO" 5 ;; esac - -fi - + ;; +esac fi - + ;; +esac fi - + ;; +esac fi - + ;; +esac fi - + ;; +esac +fi + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -17166,16 +17803,22 @@ printf %s "checking for usconfig in -lmpc... " >&6; } if test ${ac_cv_lib_mpc_usconfig+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lmpc $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char usconfig (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char usconfig (void); int main (void) { @@ -17187,12 +17830,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_mpc_usconfig=yes -else $as_nop - ac_cv_lib_mpc_usconfig=no +else case e in #( + e) ac_cv_lib_mpc_usconfig=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mpc_usconfig" >&5 printf "%s\n" "$ac_cv_lib_mpc_usconfig" >&6; } @@ -17238,12 +17883,12 @@ printf %s "checking if PTHREAD_SCOPE_SYSTEM is supported... " >&6; } if test ${ac_cv_pthread_system_supported+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_pthread_system_supported=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -17263,14 +17908,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_pthread_system_supported=yes -else $as_nop - ac_cv_pthread_system_supported=no +else case e in #( + e) ac_cv_pthread_system_supported=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_pthread_system_supported" >&5 printf "%s\n" "$ac_cv_pthread_system_supported" >&6; } @@ -17334,8 +17982,8 @@ printf "%s\n" "yes" >&6; } ipv6=yes ;; esac -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* AF_INET6 available check */ @@ -17354,10 +18002,11 @@ then : ipv6=yes -else $as_nop - +else case e in #( + e) ipv6=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext @@ -17397,12 +18046,13 @@ then : printf "%s\n" "yes" >&6; } ipv6=yes -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } ipv6=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext fi @@ -17411,7 +18061,8 @@ if test "$ipv6" = "yes"; then printf "%s\n" "#define ENABLE_IPV6 1" >>confdefs.h fi - + ;; +esac fi @@ -17430,8 +18081,8 @@ printf %s "checking for IPV6_INRIA_VERSION defined in netinet/in.h... " >&6; } if test ${ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -17452,10 +18103,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=yes -else $as_nop - ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=no +else case e in #( + e) ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&5 printf "%s\n" "$ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&6; } @@ -17471,8 +18124,8 @@ printf %s "checking for __KAME__ defined in netinet/in.h... " >&6; } if test ${ac_cv_defined___KAME___netinet_in_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -17493,10 +18146,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined___KAME___netinet_in_h=yes -else $as_nop - ac_cv_defined___KAME___netinet_in_h=no +else case e in #( + e) ac_cv_defined___KAME___netinet_in_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___KAME___netinet_in_h" >&5 printf "%s\n" "$ac_cv_defined___KAME___netinet_in_h" >&6; } @@ -17515,8 +18170,8 @@ printf %s "checking for __GLIBC__ defined in features.h... " >&6; } if test ${ac_cv_defined___GLIBC___features_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -17537,10 +18192,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined___GLIBC___features_h=yes -else $as_nop - ac_cv_defined___GLIBC___features_h=no +else case e in #( + e) ac_cv_defined___GLIBC___features_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___GLIBC___features_h" >&5 printf "%s\n" "$ac_cv_defined___GLIBC___features_h" >&6; } @@ -17573,8 +18230,8 @@ printf %s "checking for _TOSHIBA_INET6 defined in sys/param.h... " >&6; } if test ${ac_cv_defined__TOSHIBA_INET6_sys_param_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -17595,10 +18252,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined__TOSHIBA_INET6_sys_param_h=yes -else $as_nop - ac_cv_defined__TOSHIBA_INET6_sys_param_h=no +else case e in #( + e) ac_cv_defined__TOSHIBA_INET6_sys_param_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&5 printf "%s\n" "$ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&6; } @@ -17616,8 +18275,8 @@ printf %s "checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h. if test ${ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -17638,10 +18297,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=yes -else $as_nop - ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=no +else case e in #( + e) ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&5 printf "%s\n" "$ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&6; } @@ -17660,8 +18321,8 @@ printf %s "checking for _ZETA_MINAMI_INET6 defined in sys/param.h... " >&6; } if test ${ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -17682,10 +18343,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=yes -else $as_nop - ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=no +else case e in #( + e) ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&5 printf "%s\n" "$ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&6; } @@ -17721,10 +18384,11 @@ then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: libc" >&5 printf "%s\n" "libc" >&6; } -else $as_nop - +else case e in #( + e) as_fn_error $? "No $ipv6lib library found; cannot continue. You need to fetch lib$ipv6lib.a from appropriate ipv6 kit and compile beforehand." "$LINENO" 5 - + ;; +esac fi fi fi @@ -17735,8 +18399,8 @@ printf %s "checking CAN_RAW_FD_FRAMES... " >&6; } if test ${ac_cv_can_raw_fd_frames+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* CAN_RAW_FD_FRAMES available check */ @@ -17752,11 +18416,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_can_raw_fd_frames=yes -else $as_nop - ac_cv_can_raw_fd_frames=no +else case e in #( + e) ac_cv_can_raw_fd_frames=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_can_raw_fd_frames" >&5 printf "%s\n" "$ac_cv_can_raw_fd_frames" >&6; } @@ -17774,8 +18440,8 @@ printf %s "checking for CAN_RAW_JOIN_FILTERS... " >&6; } if test ${ac_cv_can_raw_join_filters+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -17791,11 +18457,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_can_raw_join_filters=yes -else $as_nop - ac_cv_can_raw_join_filters=no +else case e in #( + e) ac_cv_can_raw_join_filters=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_can_raw_join_filters" >&5 printf "%s\n" "$ac_cv_can_raw_join_filters" >&6; } @@ -17837,8 +18505,8 @@ printf %s "checking for stdatomic.h... " >&6; } if test ${ac_cv_header_stdatomic_h+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -17858,12 +18526,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_header_stdatomic_h=yes -else $as_nop - ac_cv_header_stdatomic_h=no +else case e in #( + e) ac_cv_header_stdatomic_h=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdatomic_h" >&5 printf "%s\n" "$ac_cv_header_stdatomic_h" >&6; } @@ -17883,8 +18553,8 @@ printf %s "checking for builtin __atomic_load_n and __atomic_store_n functions.. if test ${ac_cv_builtin_atomic+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -17901,12 +18571,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_builtin_atomic=yes -else $as_nop - ac_cv_builtin_atomic=no +else case e in #( + e) ac_cv_builtin_atomic=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_builtin_atomic" >&5 printf "%s\n" "$ac_cv_builtin_atomic" >&6; } @@ -17928,9 +18600,10 @@ printf %s "checking for --with-mimalloc... " >&6; } if test ${with_mimalloc+y} then : withval=$with_mimalloc; -else $as_nop - with_mimalloc="$ac_cv_header_stdatomic_h" - +else case e in #( + e) with_mimalloc="$ac_cv_header_stdatomic_h" + ;; +esac fi @@ -18019,9 +18692,10 @@ printf %s "checking for --with-valgrind... " >&6; } if test ${with_valgrind+y} then : withval=$with_valgrind; -else $as_nop - with_valgrind=no - +else case e in #( + e) with_valgrind=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_valgrind" >&5 @@ -18033,9 +18707,10 @@ then : printf "%s\n" "#define WITH_VALGRIND 1" >>confdefs.h -else $as_nop - as_fn_error $? "Valgrind support requested but headers not available" "$LINENO" 5 - +else case e in #( + e) as_fn_error $? "Valgrind support requested but headers not available" "$LINENO" 5 + ;; +esac fi OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT" @@ -18049,8 +18724,9 @@ printf %s "checking for --with-dtrace... " >&6; } if test ${with_dtrace+y} then : withval=$with_dtrace; -else $as_nop - with_dtrace=no +else case e in #( + e) with_dtrace=no ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_dtrace" >&5 @@ -18073,8 +18749,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_DTRACE+y} then : printf %s "(cached) " >&6 -else $as_nop - case $DTRACE in +else case e in #( + e) case $DTRACE in [\\/]* | ?:[\\/]*) ac_cv_path_DTRACE="$DTRACE" # Let the user override the test with a path. ;; @@ -18100,6 +18776,7 @@ IFS=$as_save_IFS test -z "$ac_cv_path_DTRACE" && ac_cv_path_DTRACE="not found" ;; +esac ;; esac fi DTRACE=$ac_cv_path_DTRACE @@ -18129,12 +18806,13 @@ printf %s "checking whether DTrace probes require linking... " >&6; } if test ${ac_cv_dtrace_link+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_dtrace_link=no +else case e in #( + e) ac_cv_dtrace_link=no echo 'BEGIN{}' > conftest.d "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ ac_cv_dtrace_link=yes - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_dtrace_link" >&5 printf "%s\n" "$ac_cv_dtrace_link" >&6; } @@ -18237,7 +18915,7 @@ if test "$ac_sys_system" = "Linux-android"; then fi for name in $blocked_funcs; do - as_func_var=`printf "%s\n" "ac_cv_func_$name" | $as_tr_sh` + as_func_var=`printf "%s\n" "ac_cv_func_$name" | sed "$as_sed_sh"` eval "$as_func_var=no" @@ -19449,8 +20127,8 @@ printf %s "checking for $CC options needed to detect all undeclared functions... if test ${ac_cv_c_undeclared_builtin_options+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_CFLAGS=$CFLAGS +else case e in #( + e) ac_save_CFLAGS=$CFLAGS ac_cv_c_undeclared_builtin_options='cannot detect' for ac_arg in '' -fno-builtin; do CFLAGS="$ac_save_CFLAGS $ac_arg" @@ -19469,8 +20147,8 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : -else $as_nop - # This test program should compile successfully. +else case e in #( + e) # This test program should compile successfully. # No library function is consistently available on # freestanding implementations, so test against a dummy # declaration. Include always-available headers on the @@ -19498,26 +20176,29 @@ then : if test x"$ac_arg" = x then : ac_cv_c_undeclared_builtin_options='none needed' -else $as_nop - ac_cv_c_undeclared_builtin_options=$ac_arg +else case e in #( + e) ac_cv_c_undeclared_builtin_options=$ac_arg ;; +esac fi break fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext done CFLAGS=$ac_save_CFLAGS - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_undeclared_builtin_options" >&5 printf "%s\n" "$ac_cv_c_undeclared_builtin_options" >&6; } case $ac_cv_c_undeclared_builtin_options in #( 'cannot detect') : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "cannot make $CC report undeclared builtins -See \`config.log' for more details" "$LINENO" 5; } ;; #( +See 'config.log' for more details" "$LINENO" 5; } ;; #( 'none needed') : ac_c_undeclared_builtin_options='' ;; #( *) : @@ -19543,8 +20224,8 @@ printf %s "checking for chroot... " >&6; } if test ${ac_cv_func_chroot+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19558,11 +20239,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_chroot=yes -else $as_nop - ac_cv_func_chroot=no +else case e in #( + e) ac_cv_func_chroot=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_chroot" >&5 printf "%s\n" "$ac_cv_func_chroot" >&6; } @@ -19582,8 +20265,8 @@ printf %s "checking for link... " >&6; } if test ${ac_cv_func_link+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19597,11 +20280,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_link=yes -else $as_nop - ac_cv_func_link=no +else case e in #( + e) ac_cv_func_link=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_link" >&5 printf "%s\n" "$ac_cv_func_link" >&6; } @@ -19621,8 +20306,8 @@ printf %s "checking for symlink... " >&6; } if test ${ac_cv_func_symlink+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19636,11 +20321,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_symlink=yes -else $as_nop - ac_cv_func_symlink=no +else case e in #( + e) ac_cv_func_symlink=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_symlink" >&5 printf "%s\n" "$ac_cv_func_symlink" >&6; } @@ -19660,8 +20347,8 @@ printf %s "checking for fchdir... " >&6; } if test ${ac_cv_func_fchdir+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19675,11 +20362,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_fchdir=yes -else $as_nop - ac_cv_func_fchdir=no +else case e in #( + e) ac_cv_func_fchdir=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_fchdir" >&5 printf "%s\n" "$ac_cv_func_fchdir" >&6; } @@ -19699,8 +20388,8 @@ printf %s "checking for fsync... " >&6; } if test ${ac_cv_func_fsync+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19714,11 +20403,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_fsync=yes -else $as_nop - ac_cv_func_fsync=no +else case e in #( + e) ac_cv_func_fsync=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_fsync" >&5 printf "%s\n" "$ac_cv_func_fsync" >&6; } @@ -19738,8 +20429,8 @@ printf %s "checking for fdatasync... " >&6; } if test ${ac_cv_func_fdatasync+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19753,11 +20444,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_fdatasync=yes -else $as_nop - ac_cv_func_fdatasync=no +else case e in #( + e) ac_cv_func_fdatasync=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_fdatasync" >&5 printf "%s\n" "$ac_cv_func_fdatasync" >&6; } @@ -19777,8 +20470,8 @@ printf %s "checking for epoll_create... " >&6; } if test ${ac_cv_func_epoll_create+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19792,11 +20485,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_epoll_create=yes -else $as_nop - ac_cv_func_epoll_create=no +else case e in #( + e) ac_cv_func_epoll_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_epoll_create" >&5 printf "%s\n" "$ac_cv_func_epoll_create" >&6; } @@ -19816,8 +20511,8 @@ printf %s "checking for epoll_create1... " >&6; } if test ${ac_cv_func_epoll_create1+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19831,11 +20526,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_epoll_create1=yes -else $as_nop - ac_cv_func_epoll_create1=no +else case e in #( + e) ac_cv_func_epoll_create1=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_epoll_create1" >&5 printf "%s\n" "$ac_cv_func_epoll_create1" >&6; } @@ -19855,8 +20552,8 @@ printf %s "checking for kqueue... " >&6; } if test ${ac_cv_func_kqueue+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -19873,11 +20570,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_kqueue=yes -else $as_nop - ac_cv_func_kqueue=no +else case e in #( + e) ac_cv_func_kqueue=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_kqueue" >&5 printf "%s\n" "$ac_cv_func_kqueue" >&6; } @@ -19897,8 +20596,8 @@ printf %s "checking for prlimit... " >&6; } if test ${ac_cv_func_prlimit+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -19915,11 +20614,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_prlimit=yes -else $as_nop - ac_cv_func_prlimit=no +else case e in #( + e) ac_cv_func_prlimit=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_prlimit" >&5 printf "%s\n" "$ac_cv_func_prlimit" >&6; } @@ -19940,8 +20641,8 @@ printf %s "checking for _dyld_shared_cache_contains_path... " >&6; } if test ${ac_cv_func__dyld_shared_cache_contains_path+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19955,11 +20656,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func__dyld_shared_cache_contains_path=yes -else $as_nop - ac_cv_func__dyld_shared_cache_contains_path=no +else case e in #( + e) ac_cv_func__dyld_shared_cache_contains_path=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func__dyld_shared_cache_contains_path" >&5 printf "%s\n" "$ac_cv_func__dyld_shared_cache_contains_path" >&6; } @@ -19980,8 +20683,8 @@ printf %s "checking for memfd_create... " >&6; } if test ${ac_cv_func_memfd_create+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef HAVE_SYS_MMAN_H @@ -20002,11 +20705,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_memfd_create=yes -else $as_nop - ac_cv_func_memfd_create=no +else case e in #( + e) ac_cv_func_memfd_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_memfd_create" >&5 printf "%s\n" "$ac_cv_func_memfd_create" >&6; } @@ -20027,8 +20732,8 @@ printf %s "checking for eventfd... " >&6; } if test ${ac_cv_func_eventfd+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef HAVE_SYS_EVENTFD_H @@ -20046,11 +20751,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_eventfd=yes -else $as_nop - ac_cv_func_eventfd=no +else case e in #( + e) ac_cv_func_eventfd=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_eventfd" >&5 printf "%s\n" "$ac_cv_func_eventfd" >&6; } @@ -20071,8 +20778,8 @@ printf %s "checking for timerfd_create... " >&6; } if test ${ac_cv_func_timerfd_create+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef HAVE_SYS_TIMERFD_H @@ -20090,11 +20797,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_timerfd_create=yes -else $as_nop - ac_cv_func_timerfd_create=no +else case e in #( + e) ac_cv_func_timerfd_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_timerfd_create" >&5 printf "%s\n" "$ac_cv_func_timerfd_create" >&6; } @@ -20121,8 +20830,8 @@ printf %s "checking for ctermid_r... " >&6; } if test ${ac_cv_func_ctermid_r+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -20136,11 +20845,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_ctermid_r=yes -else $as_nop - ac_cv_func_ctermid_r=no +else case e in #( + e) ac_cv_func_ctermid_r=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_ctermid_r" >&5 printf "%s\n" "$ac_cv_func_ctermid_r" >&6; } @@ -20159,8 +20870,8 @@ printf %s "checking for flock declaration... " >&6; } if test ${ac_cv_flock_decl+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -20175,12 +20886,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_flock_decl=yes -else $as_nop - ac_cv_flock_decl=no - +else case e in #( + e) ac_cv_flock_decl=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_flock_decl" >&5 printf "%s\n" "$ac_cv_flock_decl" >&6; } @@ -20194,22 +20907,28 @@ if test "x$ac_cv_func_flock" = xyes then : printf "%s\n" "#define HAVE_FLOCK 1" >>confdefs.h -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for flock in -lbsd" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for flock in -lbsd" >&5 printf %s "checking for flock in -lbsd... " >&6; } if test ${ac_cv_lib_bsd_flock+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lbsd $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char flock (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char flock (void); int main (void) { @@ -20221,12 +20940,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_bsd_flock=yes -else $as_nop - ac_cv_lib_bsd_flock=no +else case e in #( + e) ac_cv_lib_bsd_flock=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bsd_flock" >&5 printf "%s\n" "$ac_cv_lib_bsd_flock" >&6; } @@ -20234,7 +20955,8 @@ if test "x$ac_cv_lib_bsd_flock" = xyes then : FCNTL_LIBS="-lbsd" fi - + ;; +esac fi done @@ -20247,8 +20969,8 @@ printf %s "checking for getpagesize... " >&6; } if test ${ac_cv_func_getpagesize+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -20262,11 +20984,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getpagesize=yes -else $as_nop - ac_cv_func_getpagesize=no +else case e in #( + e) ac_cv_func_getpagesize=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getpagesize" >&5 printf "%s\n" "$ac_cv_func_getpagesize" >&6; } @@ -20285,8 +21009,8 @@ printf %s "checking for broken unsetenv... " >&6; } if test ${ac_cv_broken_unsetenv+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -20300,12 +21024,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_broken_unsetenv=no -else $as_nop - ac_cv_broken_unsetenv=yes - +else case e in #( + e) ac_cv_broken_unsetenv=yes + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_broken_unsetenv" >&5 printf "%s\n" "$ac_cv_broken_unsetenv" >&6; } @@ -20327,8 +21053,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_TRUE+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$TRUE"; then +else case e in #( + e) if test -n "$TRUE"; then ac_cv_prog_TRUE="$TRUE" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -20350,7 +21076,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi TRUE=$ac_cv_prog_TRUE if test -n "$TRUE"; then @@ -20372,16 +21099,22 @@ printf %s "checking for inet_aton in -lc... " >&6; } if test ${ac_cv_lib_c_inet_aton+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lc $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char inet_aton (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char inet_aton (void); int main (void) { @@ -20393,34 +21126,42 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_c_inet_aton=yes -else $as_nop - ac_cv_lib_c_inet_aton=no +else case e in #( + e) ac_cv_lib_c_inet_aton=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_c_inet_aton" >&5 printf "%s\n" "$ac_cv_lib_c_inet_aton" >&6; } if test "x$ac_cv_lib_c_inet_aton" = xyes then : $ac_cv_prog_TRUE -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for inet_aton in -lresolv" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for inet_aton in -lresolv" >&5 printf %s "checking for inet_aton in -lresolv... " >&6; } if test ${ac_cv_lib_resolv_inet_aton+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lresolv $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char inet_aton (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char inet_aton (void); int main (void) { @@ -20432,12 +21173,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_resolv_inet_aton=yes -else $as_nop - ac_cv_lib_resolv_inet_aton=no +else case e in #( + e) ac_cv_lib_resolv_inet_aton=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_resolv_inet_aton" >&5 printf "%s\n" "$ac_cv_lib_resolv_inet_aton" >&6; } @@ -20446,7 +21189,8 @@ then : SOCKET_LIBS="-lresolv" fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for hstrerror in -lc" >&5 @@ -20454,16 +21198,22 @@ printf %s "checking for hstrerror in -lc... " >&6; } if test ${ac_cv_lib_c_hstrerror+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lc $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char hstrerror (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char hstrerror (void); int main (void) { @@ -20475,34 +21225,42 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_c_hstrerror=yes -else $as_nop - ac_cv_lib_c_hstrerror=no +else case e in #( + e) ac_cv_lib_c_hstrerror=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_c_hstrerror" >&5 printf "%s\n" "$ac_cv_lib_c_hstrerror" >&6; } if test "x$ac_cv_lib_c_hstrerror" = xyes then : $ac_cv_prog_TRUE -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for hstrerror in -lresolv" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for hstrerror in -lresolv" >&5 printf %s "checking for hstrerror in -lresolv... " >&6; } if test ${ac_cv_lib_resolv_hstrerror+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lresolv $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char hstrerror (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char hstrerror (void); int main (void) { @@ -20514,12 +21272,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_resolv_hstrerror=yes -else $as_nop - ac_cv_lib_resolv_hstrerror=no +else case e in #( + e) ac_cv_lib_resolv_hstrerror=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_resolv_hstrerror" >&5 printf "%s\n" "$ac_cv_lib_resolv_hstrerror" >&6; } @@ -20528,7 +21288,8 @@ then : SOCKET_LIBS="-lresolv" fi - + ;; +esac fi @@ -20539,12 +21300,12 @@ printf %s "checking for chflags... " >&6; } if test ${ac_cv_have_chflags+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_have_chflags=cross -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -20560,14 +21321,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_have_chflags=yes -else $as_nop - ac_cv_have_chflags=no +else case e in #( + e) ac_cv_have_chflags=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_chflags" >&5 printf "%s\n" "$ac_cv_have_chflags" >&6; } @@ -20576,8 +21340,9 @@ if test "$ac_cv_have_chflags" = cross ; then if test "x$ac_cv_func_chflags" = xyes then : ac_cv_have_chflags="yes" -else $as_nop - ac_cv_have_chflags="no" +else case e in #( + e) ac_cv_have_chflags="no" ;; +esac fi fi @@ -20592,12 +21357,12 @@ printf %s "checking for lchflags... " >&6; } if test ${ac_cv_have_lchflags+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_have_lchflags=cross -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -20613,14 +21378,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_have_lchflags=yes -else $as_nop - ac_cv_have_lchflags=no +else case e in #( + e) ac_cv_have_lchflags=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_lchflags" >&5 printf "%s\n" "$ac_cv_have_lchflags" >&6; } @@ -20629,8 +21397,9 @@ if test "$ac_cv_have_lchflags" = cross ; then if test "x$ac_cv_func_lchflags" = xyes then : ac_cv_have_lchflags="yes" -else $as_nop - ac_cv_have_lchflags="no" +else case e in #( + e) ac_cv_have_lchflags="no" ;; +esac fi fi @@ -20737,16 +21506,22 @@ printf %s "checking for gzread in -lz... " >&6; } if test ${ac_cv_lib_z_gzread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lz $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char gzread (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char gzread (void); int main (void) { @@ -20758,27 +21533,31 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_z_gzread=yes -else $as_nop - ac_cv_lib_z_gzread=no +else case e in #( + e) ac_cv_lib_z_gzread=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_z_gzread" >&5 printf "%s\n" "$ac_cv_lib_z_gzread" >&6; } if test "x$ac_cv_lib_z_gzread" = xyes then : have_zlib=yes -else $as_nop - have_zlib=no +else case e in #( + e) have_zlib=no ;; +esac fi LIBS=$py_check_lib_save_LIBS -else $as_nop - have_zlib=no +else case e in #( + e) have_zlib=no ;; +esac fi done @@ -20793,16 +21572,22 @@ printf %s "checking for inflateCopy in -lz... " >&6; } if test ${ac_cv_lib_z_inflateCopy+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lz $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char inflateCopy (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char inflateCopy (void); int main (void) { @@ -20814,12 +21599,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_z_inflateCopy=yes -else $as_nop - ac_cv_lib_z_inflateCopy=no +else case e in #( + e) ac_cv_lib_z_inflateCopy=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_z_inflateCopy" >&5 printf "%s\n" "$ac_cv_lib_z_inflateCopy" >&6; } @@ -20866,16 +21653,22 @@ printf %s "checking for gzread in -lz... " >&6; } if test ${ac_cv_lib_z_gzread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lz $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char gzread (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char gzread (void); int main (void) { @@ -20887,27 +21680,31 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_z_gzread=yes -else $as_nop - ac_cv_lib_z_gzread=no +else case e in #( + e) ac_cv_lib_z_gzread=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_z_gzread" >&5 printf "%s\n" "$ac_cv_lib_z_gzread" >&6; } if test "x$ac_cv_lib_z_gzread" = xyes then : have_zlib=yes -else $as_nop - have_zlib=no +else case e in #( + e) have_zlib=no ;; +esac fi LIBS=$py_check_lib_save_LIBS -else $as_nop - have_zlib=no +else case e in #( + e) have_zlib=no ;; +esac fi done @@ -20922,16 +21719,22 @@ printf %s "checking for inflateCopy in -lz... " >&6; } if test ${ac_cv_lib_z_inflateCopy+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lz $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char inflateCopy (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char inflateCopy (void); int main (void) { @@ -20943,12 +21746,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_z_inflateCopy=yes -else $as_nop - ac_cv_lib_z_inflateCopy=no +else case e in #( + e) ac_cv_lib_z_inflateCopy=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_z_inflateCopy" >&5 printf "%s\n" "$ac_cv_lib_z_inflateCopy" >&6; } @@ -21084,16 +21889,22 @@ printf %s "checking for BZ2_bzCompress in -lbz2... " >&6; } if test ${ac_cv_lib_bz2_BZ2_bzCompress+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lbz2 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char BZ2_bzCompress (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char BZ2_bzCompress (void); int main (void) { @@ -21105,25 +21916,29 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_bz2_BZ2_bzCompress=yes -else $as_nop - ac_cv_lib_bz2_BZ2_bzCompress=no +else case e in #( + e) ac_cv_lib_bz2_BZ2_bzCompress=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bz2_BZ2_bzCompress" >&5 printf "%s\n" "$ac_cv_lib_bz2_BZ2_bzCompress" >&6; } if test "x$ac_cv_lib_bz2_BZ2_bzCompress" = xyes then : have_bzip2=yes -else $as_nop - have_bzip2=no +else case e in #( + e) have_bzip2=no ;; +esac fi -else $as_nop - have_bzip2=no +else case e in #( + e) have_bzip2=no ;; +esac fi done @@ -21166,16 +21981,22 @@ printf %s "checking for BZ2_bzCompress in -lbz2... " >&6; } if test ${ac_cv_lib_bz2_BZ2_bzCompress+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lbz2 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char BZ2_bzCompress (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char BZ2_bzCompress (void); int main (void) { @@ -21187,25 +22008,29 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_bz2_BZ2_bzCompress=yes -else $as_nop - ac_cv_lib_bz2_BZ2_bzCompress=no +else case e in #( + e) ac_cv_lib_bz2_BZ2_bzCompress=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bz2_BZ2_bzCompress" >&5 printf "%s\n" "$ac_cv_lib_bz2_BZ2_bzCompress" >&6; } if test "x$ac_cv_lib_bz2_BZ2_bzCompress" = xyes then : have_bzip2=yes -else $as_nop - have_bzip2=no +else case e in #( + e) have_bzip2=no ;; +esac fi -else $as_nop - have_bzip2=no +else case e in #( + e) have_bzip2=no ;; +esac fi done @@ -21312,16 +22137,22 @@ printf %s "checking for lzma_easy_encoder in -llzma... " >&6; } if test ${ac_cv_lib_lzma_lzma_easy_encoder+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-llzma $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char lzma_easy_encoder (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char lzma_easy_encoder (void); int main (void) { @@ -21333,25 +22164,29 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_lzma_lzma_easy_encoder=yes -else $as_nop - ac_cv_lib_lzma_lzma_easy_encoder=no +else case e in #( + e) ac_cv_lib_lzma_lzma_easy_encoder=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzma_lzma_easy_encoder" >&5 printf "%s\n" "$ac_cv_lib_lzma_lzma_easy_encoder" >&6; } if test "x$ac_cv_lib_lzma_lzma_easy_encoder" = xyes then : have_liblzma=yes -else $as_nop - have_liblzma=no +else case e in #( + e) have_liblzma=no ;; +esac fi -else $as_nop - have_liblzma=no +else case e in #( + e) have_liblzma=no ;; +esac fi done @@ -21394,16 +22229,22 @@ printf %s "checking for lzma_easy_encoder in -llzma... " >&6; } if test ${ac_cv_lib_lzma_lzma_easy_encoder+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-llzma $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char lzma_easy_encoder (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char lzma_easy_encoder (void); int main (void) { @@ -21415,25 +22256,29 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_lzma_lzma_easy_encoder=yes -else $as_nop - ac_cv_lib_lzma_lzma_easy_encoder=no +else case e in #( + e) ac_cv_lib_lzma_lzma_easy_encoder=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzma_lzma_easy_encoder" >&5 printf "%s\n" "$ac_cv_lib_lzma_lzma_easy_encoder" >&6; } if test "x$ac_cv_lib_lzma_lzma_easy_encoder" = xyes then : have_liblzma=yes -else $as_nop - have_liblzma=no +else case e in #( + e) have_liblzma=no ;; +esac fi -else $as_nop - have_liblzma=no +else case e in #( + e) have_liblzma=no ;; +esac fi done @@ -21469,8 +22314,8 @@ printf %s "checking for hstrerror... " >&6; } if test ${ac_cv_func_hstrerror+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -21484,11 +22329,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_hstrerror=yes -else $as_nop - ac_cv_func_hstrerror=no +else case e in #( + e) ac_cv_func_hstrerror=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_hstrerror" >&5 printf "%s\n" "$ac_cv_func_hstrerror" >&6; } @@ -21508,8 +22355,8 @@ printf %s "checking for getservbyname... " >&6; } if test ${ac_cv_func_getservbyname+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -21523,11 +22370,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getservbyname=yes -else $as_nop - ac_cv_func_getservbyname=no +else case e in #( + e) ac_cv_func_getservbyname=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getservbyname" >&5 printf "%s\n" "$ac_cv_func_getservbyname" >&6; } @@ -21547,8 +22396,8 @@ printf %s "checking for getservbyport... " >&6; } if test ${ac_cv_func_getservbyport+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -21562,11 +22411,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getservbyport=yes -else $as_nop - ac_cv_func_getservbyport=no +else case e in #( + e) ac_cv_func_getservbyport=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getservbyport" >&5 printf "%s\n" "$ac_cv_func_getservbyport" >&6; } @@ -21586,8 +22437,8 @@ printf %s "checking for gethostbyname... " >&6; } if test ${ac_cv_func_gethostbyname+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -21601,11 +22452,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_gethostbyname=yes -else $as_nop - ac_cv_func_gethostbyname=no +else case e in #( + e) ac_cv_func_gethostbyname=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_gethostbyname" >&5 printf "%s\n" "$ac_cv_func_gethostbyname" >&6; } @@ -21625,8 +22478,8 @@ printf %s "checking for gethostbyaddr... " >&6; } if test ${ac_cv_func_gethostbyaddr+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -21640,11 +22493,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_gethostbyaddr=yes -else $as_nop - ac_cv_func_gethostbyaddr=no +else case e in #( + e) ac_cv_func_gethostbyaddr=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_gethostbyaddr" >&5 printf "%s\n" "$ac_cv_func_gethostbyaddr" >&6; } @@ -21664,8 +22519,8 @@ printf %s "checking for getprotobyname... " >&6; } if test ${ac_cv_func_getprotobyname+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -21679,11 +22534,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getprotobyname=yes -else $as_nop - ac_cv_func_getprotobyname=no +else case e in #( + e) ac_cv_func_getprotobyname=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getprotobyname" >&5 printf "%s\n" "$ac_cv_func_getprotobyname" >&6; } @@ -21706,8 +22563,8 @@ printf %s "checking for inet_aton... " >&6; } if test ${ac_cv_func_inet_aton+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21726,11 +22583,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_inet_aton=yes -else $as_nop - ac_cv_func_inet_aton=no +else case e in #( + e) ac_cv_func_inet_aton=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_inet_aton" >&5 printf "%s\n" "$ac_cv_func_inet_aton" >&6; } @@ -21750,8 +22609,8 @@ printf %s "checking for inet_ntoa... " >&6; } if test ${ac_cv_func_inet_ntoa+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21770,11 +22629,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_inet_ntoa=yes -else $as_nop - ac_cv_func_inet_ntoa=no +else case e in #( + e) ac_cv_func_inet_ntoa=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_inet_ntoa" >&5 printf "%s\n" "$ac_cv_func_inet_ntoa" >&6; } @@ -21794,8 +22655,8 @@ printf %s "checking for inet_pton... " >&6; } if test ${ac_cv_func_inet_pton+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21814,11 +22675,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_inet_pton=yes -else $as_nop - ac_cv_func_inet_pton=no +else case e in #( + e) ac_cv_func_inet_pton=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_inet_pton" >&5 printf "%s\n" "$ac_cv_func_inet_pton" >&6; } @@ -21838,8 +22701,8 @@ printf %s "checking for getpeername... " >&6; } if test ${ac_cv_func_getpeername+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21858,11 +22721,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getpeername=yes -else $as_nop - ac_cv_func_getpeername=no +else case e in #( + e) ac_cv_func_getpeername=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getpeername" >&5 printf "%s\n" "$ac_cv_func_getpeername" >&6; } @@ -21882,8 +22747,8 @@ printf %s "checking for getsockname... " >&6; } if test ${ac_cv_func_getsockname+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21902,11 +22767,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getsockname=yes -else $as_nop - ac_cv_func_getsockname=no +else case e in #( + e) ac_cv_func_getsockname=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getsockname" >&5 printf "%s\n" "$ac_cv_func_getsockname" >&6; } @@ -21926,8 +22793,8 @@ printf %s "checking for accept... " >&6; } if test ${ac_cv_func_accept+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21946,11 +22813,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_accept=yes -else $as_nop - ac_cv_func_accept=no +else case e in #( + e) ac_cv_func_accept=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_accept" >&5 printf "%s\n" "$ac_cv_func_accept" >&6; } @@ -21970,8 +22839,8 @@ printf %s "checking for bind... " >&6; } if test ${ac_cv_func_bind+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21990,11 +22859,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_bind=yes -else $as_nop - ac_cv_func_bind=no +else case e in #( + e) ac_cv_func_bind=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_bind" >&5 printf "%s\n" "$ac_cv_func_bind" >&6; } @@ -22014,8 +22885,8 @@ printf %s "checking for connect... " >&6; } if test ${ac_cv_func_connect+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -22034,11 +22905,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_connect=yes -else $as_nop - ac_cv_func_connect=no +else case e in #( + e) ac_cv_func_connect=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_connect" >&5 printf "%s\n" "$ac_cv_func_connect" >&6; } @@ -22058,8 +22931,8 @@ printf %s "checking for listen... " >&6; } if test ${ac_cv_func_listen+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -22078,11 +22951,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_listen=yes -else $as_nop - ac_cv_func_listen=no +else case e in #( + e) ac_cv_func_listen=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_listen" >&5 printf "%s\n" "$ac_cv_func_listen" >&6; } @@ -22102,8 +22977,8 @@ printf %s "checking for recvfrom... " >&6; } if test ${ac_cv_func_recvfrom+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -22122,11 +22997,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_recvfrom=yes -else $as_nop - ac_cv_func_recvfrom=no +else case e in #( + e) ac_cv_func_recvfrom=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_recvfrom" >&5 printf "%s\n" "$ac_cv_func_recvfrom" >&6; } @@ -22146,8 +23023,8 @@ printf %s "checking for sendto... " >&6; } if test ${ac_cv_func_sendto+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -22166,11 +23043,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_sendto=yes -else $as_nop - ac_cv_func_sendto=no +else case e in #( + e) ac_cv_func_sendto=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_sendto" >&5 printf "%s\n" "$ac_cv_func_sendto" >&6; } @@ -22190,8 +23069,8 @@ printf %s "checking for setsockopt... " >&6; } if test ${ac_cv_func_setsockopt+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -22210,11 +23089,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_setsockopt=yes -else $as_nop - ac_cv_func_setsockopt=no +else case e in #( + e) ac_cv_func_setsockopt=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_setsockopt" >&5 printf "%s\n" "$ac_cv_func_setsockopt" >&6; } @@ -22234,8 +23115,8 @@ printf %s "checking for socket... " >&6; } if test ${ac_cv_func_socket+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -22254,11 +23135,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_socket=yes -else $as_nop - ac_cv_func_socket=no +else case e in #( + e) ac_cv_func_socket=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_socket" >&5 printf "%s\n" "$ac_cv_func_socket" >&6; } @@ -22280,8 +23163,8 @@ printf %s "checking for setgroups... " >&6; } if test ${ac_cv_func_setgroups+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -22300,11 +23183,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_setgroups=yes -else $as_nop - ac_cv_func_setgroups=no +else case e in #( + e) ac_cv_func_setgroups=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_setgroups" >&5 printf "%s\n" "$ac_cv_func_setgroups" >&6; } @@ -22328,22 +23213,28 @@ if test "x$ac_cv_func_openpty" = xyes then : printf "%s\n" "#define HAVE_OPENPTY 1" >>confdefs.h -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for openpty in -lutil" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for openpty in -lutil" >&5 printf %s "checking for openpty in -lutil... " >&6; } if test ${ac_cv_lib_util_openpty+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lutil $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char openpty (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char openpty (void); int main (void) { @@ -22355,12 +23246,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_util_openpty=yes -else $as_nop - ac_cv_lib_util_openpty=no +else case e in #( + e) ac_cv_lib_util_openpty=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_util_openpty" >&5 printf "%s\n" "$ac_cv_lib_util_openpty" >&6; } @@ -22368,22 +23261,28 @@ if test "x$ac_cv_lib_util_openpty" = xyes then : printf "%s\n" "#define HAVE_OPENPTY 1" >>confdefs.h LIBS="$LIBS -lutil" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for openpty in -lbsd" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for openpty in -lbsd" >&5 printf %s "checking for openpty in -lbsd... " >&6; } if test ${ac_cv_lib_bsd_openpty+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lbsd $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char openpty (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char openpty (void); int main (void) { @@ -22395,12 +23294,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_bsd_openpty=yes -else $as_nop - ac_cv_lib_bsd_openpty=no +else case e in #( + e) ac_cv_lib_bsd_openpty=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bsd_openpty" >&5 printf "%s\n" "$ac_cv_lib_bsd_openpty" >&6; } @@ -22409,9 +23310,11 @@ then : printf "%s\n" "#define HAVE_OPENPTY 1" >>confdefs.h LIBS="$LIBS -lbsd" fi - + ;; +esac fi - + ;; +esac fi done @@ -22420,15 +23323,21 @@ printf %s "checking for library containing login_tty... " >&6; } if test ${ac_cv_search_login_tty+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char login_tty (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char login_tty (void); int main (void) { @@ -22459,11 +23368,13 @@ done if test ${ac_cv_search_login_tty+y} then : -else $as_nop - ac_cv_search_login_tty=no +else case e in #( + e) ac_cv_search_login_tty=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_login_tty" >&5 printf "%s\n" "$ac_cv_search_login_tty" >&6; } @@ -22485,22 +23396,28 @@ if test "x$ac_cv_func_forkpty" = xyes then : printf "%s\n" "#define HAVE_FORKPTY 1" >>confdefs.h -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for forkpty in -lutil" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for forkpty in -lutil" >&5 printf %s "checking for forkpty in -lutil... " >&6; } if test ${ac_cv_lib_util_forkpty+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lutil $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char forkpty (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char forkpty (void); int main (void) { @@ -22512,12 +23429,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_util_forkpty=yes -else $as_nop - ac_cv_lib_util_forkpty=no +else case e in #( + e) ac_cv_lib_util_forkpty=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_util_forkpty" >&5 printf "%s\n" "$ac_cv_lib_util_forkpty" >&6; } @@ -22525,22 +23444,28 @@ if test "x$ac_cv_lib_util_forkpty" = xyes then : printf "%s\n" "#define HAVE_FORKPTY 1" >>confdefs.h LIBS="$LIBS -lutil" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for forkpty in -lbsd" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for forkpty in -lbsd" >&5 printf %s "checking for forkpty in -lbsd... " >&6; } if test ${ac_cv_lib_bsd_forkpty+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lbsd $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char forkpty (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char forkpty (void); int main (void) { @@ -22552,12 +23477,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_bsd_forkpty=yes -else $as_nop - ac_cv_lib_bsd_forkpty=no +else case e in #( + e) ac_cv_lib_bsd_forkpty=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bsd_forkpty" >&5 printf "%s\n" "$ac_cv_lib_bsd_forkpty" >&6; } @@ -22566,9 +23493,11 @@ then : printf "%s\n" "#define HAVE_FORKPTY 1" >>confdefs.h LIBS="$LIBS -lbsd" fi - + ;; +esac fi - + ;; +esac fi done @@ -22617,13 +23546,14 @@ if test "x$ac_cv_func_dup2" = xyes then : printf "%s\n" "#define HAVE_DUP2 1" >>confdefs.h -else $as_nop - case " $LIBOBJS " in +else case e in #( + e) case " $LIBOBJS " in *" dup2.$ac_objext "* ) ;; *) LIBOBJS="$LIBOBJS dup2.$ac_objext" ;; esac - + ;; +esac fi @@ -22706,23 +23636,29 @@ if test "x$ac_cv_func_clock_gettime" = xyes then : printf "%s\n" "#define HAVE_CLOCK_GETTIME 1" >>confdefs.h -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_gettime in -lrt" >&5 printf %s "checking for clock_gettime in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_gettime+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lrt $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char clock_gettime (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char clock_gettime (void); int main (void) { @@ -22734,12 +23670,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_rt_clock_gettime=yes -else $as_nop - ac_cv_lib_rt_clock_gettime=no +else case e in #( + e) ac_cv_lib_rt_clock_gettime=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_gettime" >&5 printf "%s\n" "$ac_cv_lib_rt_clock_gettime" >&6; } @@ -22755,7 +23693,8 @@ printf "%s\n" "#define TIMEMODULE_LIB rt" >>confdefs.h fi - + ;; +esac fi done @@ -22768,23 +23707,29 @@ if test "x$ac_cv_func_clock_getres" = xyes then : printf "%s\n" "#define HAVE_CLOCK_GETRES 1" >>confdefs.h -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_getres in -lrt" >&5 printf %s "checking for clock_getres in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_getres+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lrt $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char clock_getres (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char clock_getres (void); int main (void) { @@ -22796,12 +23741,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_rt_clock_getres=yes -else $as_nop - ac_cv_lib_rt_clock_getres=no +else case e in #( + e) ac_cv_lib_rt_clock_getres=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_getres" >&5 printf "%s\n" "$ac_cv_lib_rt_clock_getres" >&6; } @@ -22813,7 +23760,8 @@ then : fi - + ;; +esac fi done @@ -22831,23 +23779,29 @@ if test "x$ac_cv_func_clock_settime" = xyes then : printf "%s\n" "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 printf %s "checking for clock_settime in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_settime+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lrt $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char clock_settime (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char clock_settime (void); int main (void) { @@ -22859,12 +23813,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_rt_clock_settime=yes -else $as_nop - ac_cv_lib_rt_clock_settime=no +else case e in #( + e) ac_cv_lib_rt_clock_settime=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_settime" >&5 printf "%s\n" "$ac_cv_lib_rt_clock_settime" >&6; } @@ -22876,7 +23832,8 @@ then : fi - + ;; +esac fi done @@ -22894,23 +23851,29 @@ if test "x$ac_cv_func_clock_nanosleep" = xyes then : printf "%s\n" "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 printf %s "checking for clock_nanosleep in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_nanosleep+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lrt $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char clock_nanosleep (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char clock_nanosleep (void); int main (void) { @@ -22922,12 +23885,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_rt_clock_nanosleep=yes -else $as_nop - ac_cv_lib_rt_clock_nanosleep=no +else case e in #( + e) ac_cv_lib_rt_clock_nanosleep=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_nanosleep" >&5 printf "%s\n" "$ac_cv_lib_rt_clock_nanosleep" >&6; } @@ -22939,7 +23904,8 @@ then : fi - + ;; +esac fi done @@ -22953,23 +23919,29 @@ if test "x$ac_cv_func_nanosleep" = xyes then : printf "%s\n" "#define HAVE_NANOSLEEP 1" >>confdefs.h -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for nanosleep in -lrt" >&5 printf %s "checking for nanosleep in -lrt... " >&6; } if test ${ac_cv_lib_rt_nanosleep+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lrt $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char nanosleep (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char nanosleep (void); int main (void) { @@ -22981,12 +23953,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_rt_nanosleep=yes -else $as_nop - ac_cv_lib_rt_nanosleep=no +else case e in #( + e) ac_cv_lib_rt_nanosleep=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_nanosleep" >&5 printf "%s\n" "$ac_cv_lib_rt_nanosleep" >&6; } @@ -22998,7 +23972,8 @@ then : fi - + ;; +esac fi done @@ -23008,8 +23983,8 @@ printf %s "checking for major, minor, and makedev... " >&6; } if test ${ac_cv_device_macros+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -23035,12 +24010,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_device_macros=yes -else $as_nop - ac_cv_device_macros=no +else case e in #( + e) ac_cv_device_macros=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_device_macros" >&5 printf "%s\n" "$ac_cv_device_macros" >&6; } @@ -23064,8 +24041,8 @@ printf %s "checking for getaddrinfo... " >&6; } if test ${ac_cv_func_getaddrinfo+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -23085,12 +24062,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_func_getaddrinfo=yes -else $as_nop - ac_cv_func_getaddrinfo=no +else case e in #( + e) ac_cv_func_getaddrinfo=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getaddrinfo" >&5 printf "%s\n" "$ac_cv_func_getaddrinfo" >&6; } @@ -23103,8 +24082,8 @@ printf %s "checking getaddrinfo bug... " >&6; } if test ${ac_cv_buggy_getaddrinfo+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then @@ -23114,8 +24093,8 @@ elif test "${enable_ipv6+set}" = set; then else ac_cv_buggy_getaddrinfo=yes fi -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -23211,13 +24190,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_buggy_getaddrinfo=no -else $as_nop - ac_cv_buggy_getaddrinfo=yes +else case e in #( + e) ac_cv_buggy_getaddrinfo=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_buggy_getaddrinfo" >&5 printf "%s\n" "$ac_cv_buggy_getaddrinfo" >&6; } @@ -23253,8 +24235,8 @@ printf %s "checking whether struct tm is in sys/time.h or time.h... " >&6; } if test ${ac_cv_struct_tm+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include @@ -23272,10 +24254,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_struct_tm=time.h -else $as_nop - ac_cv_struct_tm=sys/time.h +else case e in #( + e) ac_cv_struct_tm=sys/time.h ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_struct_tm" >&5 printf "%s\n" "$ac_cv_struct_tm" >&6; } @@ -23307,8 +24291,9 @@ else if test "x$ac_cv_have_decl_tzname" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_TZNAME $ac_have_decl" >>confdefs.h @@ -23317,8 +24302,8 @@ printf %s "checking for tzname... " >&6; } if test ${ac_cv_var_tzname+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #if !HAVE_DECL_TZNAME @@ -23336,11 +24321,13 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_var_tzname=yes -else $as_nop - ac_cv_var_tzname=no +else case e in #( + e) ac_cv_var_tzname=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext + conftest$ac_exeext conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_var_tzname" >&5 printf "%s\n" "$ac_cv_var_tzname" >&6; } @@ -23447,8 +24434,8 @@ printf %s "checking for time.h that defines altzone... " >&6; } if test ${ac_cv_header_time_altzone+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -23463,11 +24450,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_header_time_altzone=yes -else $as_nop - ac_cv_header_time_altzone=no +else case e in #( + e) ac_cv_header_time_altzone=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_time_altzone" >&5 printf "%s\n" "$ac_cv_header_time_altzone" >&6; } @@ -23482,8 +24471,8 @@ printf %s "checking for addrinfo... " >&6; } if test ${ac_cv_struct_addrinfo+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -23497,10 +24486,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_struct_addrinfo=yes -else $as_nop - ac_cv_struct_addrinfo=no +else case e in #( + e) ac_cv_struct_addrinfo=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_struct_addrinfo" >&5 printf "%s\n" "$ac_cv_struct_addrinfo" >&6; } @@ -23515,8 +24506,8 @@ printf %s "checking for sockaddr_storage... " >&6; } if test ${ac_cv_struct_sockaddr_storage+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ # include @@ -23532,10 +24523,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_struct_sockaddr_storage=yes -else $as_nop - ac_cv_struct_sockaddr_storage=no +else case e in #( + e) ac_cv_struct_sockaddr_storage=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_struct_sockaddr_storage" >&5 printf "%s\n" "$ac_cv_struct_sockaddr_storage" >&6; } @@ -23550,8 +24543,8 @@ printf %s "checking for sockaddr_alg... " >&6; } if test ${ac_cv_struct_sockaddr_alg+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ # include @@ -23568,10 +24561,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_struct_sockaddr_alg=yes -else $as_nop - ac_cv_struct_sockaddr_alg=no +else case e in #( + e) ac_cv_struct_sockaddr_alg=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_struct_sockaddr_alg" >&5 printf "%s\n" "$ac_cv_struct_sockaddr_alg" >&6; } @@ -23588,8 +24583,8 @@ printf %s "checking for an ANSI C-conforming const... " >&6; } if test ${ac_cv_c_const+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -23653,10 +24648,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_c_const=yes -else $as_nop - ac_cv_c_const=no +else case e in #( + e) ac_cv_c_const=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_const" >&5 printf "%s\n" "$ac_cv_c_const" >&6; } @@ -23672,8 +24669,8 @@ printf %s "checking for working signed char... " >&6; } if test ${ac_cv_working_signed_char_c+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -23688,11 +24685,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_working_signed_char_c=yes -else $as_nop - ac_cv_working_signed_char_c=no +else case e in #( + e) ac_cv_working_signed_char_c=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_signed_char_c" >&5 printf "%s\n" "$ac_cv_working_signed_char_c" >&6; } @@ -23710,8 +24709,8 @@ printf %s "checking for prototypes... " >&6; } if test ${ac_cv_function_prototypes+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int foo(int x) { return 0; } @@ -23726,11 +24725,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_function_prototypes=yes -else $as_nop - ac_cv_function_prototypes=no +else case e in #( + e) ac_cv_function_prototypes=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_function_prototypes" >&5 printf "%s\n" "$ac_cv_function_prototypes" >&6; } @@ -23752,8 +24753,8 @@ printf %s "checking for socketpair... " >&6; } if test ${ac_cv_func_socketpair+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -23770,11 +24771,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_socketpair=yes -else $as_nop - ac_cv_func_socketpair=no +else case e in #( + e) ac_cv_func_socketpair=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_socketpair" >&5 printf "%s\n" "$ac_cv_func_socketpair" >&6; } @@ -23794,8 +24797,8 @@ printf %s "checking if sockaddr has sa_len member... " >&6; } if test ${ac_cv_struct_sockaddr_sa_len+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -23812,11 +24815,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_struct_sockaddr_sa_len=yes -else $as_nop - ac_cv_struct_sockaddr_sa_len=no +else case e in #( + e) ac_cv_struct_sockaddr_sa_len=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_struct_sockaddr_sa_len" >&5 printf "%s\n" "$ac_cv_struct_sockaddr_sa_len" >&6; } @@ -23873,8 +24878,8 @@ printf "%s\n" "#define HAVE_GETHOSTBYNAME_R_6_ARG 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking gethostbyname_r with 5 args" >&5 @@ -23911,8 +24916,8 @@ printf "%s\n" "#define HAVE_GETHOSTBYNAME_R_5_ARG 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking gethostbyname_r with 3 args" >&5 @@ -23947,23 +24952,26 @@ printf "%s\n" "#define HAVE_GETHOSTBYNAME_R_3_ARG 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$OLD_CFLAGS -else $as_nop - +else case e in #( + e) ac_fn_c_check_func "$LINENO" "gethostbyname" "ac_cv_func_gethostbyname" if test "x$ac_cv_func_gethostbyname" = xyes then : @@ -23971,7 +24979,8 @@ then : fi - + ;; +esac fi @@ -23988,22 +24997,28 @@ ac_fn_c_check_func "$LINENO" "__fpu_control" "ac_cv_func___fpu_control" if test "x$ac_cv_func___fpu_control" = xyes then : -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __fpu_control in -lieee" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __fpu_control in -lieee" >&5 printf %s "checking for __fpu_control in -lieee... " >&6; } if test ${ac_cv_lib_ieee___fpu_control+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lieee $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char __fpu_control (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char __fpu_control (void); int main (void) { @@ -24015,12 +25030,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_ieee___fpu_control=yes -else $as_nop - ac_cv_lib_ieee___fpu_control=no +else case e in #( + e) ac_cv_lib_ieee___fpu_control=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ieee___fpu_control" >&5 printf "%s\n" "$ac_cv_lib_ieee___fpu_control" >&6; } @@ -24032,7 +25049,8 @@ then : fi - + ;; +esac fi @@ -24059,9 +25077,10 @@ then LIBM=$withval printf "%s\n" "set LIBM=\"$withval\"" >&6; } else as_fn_error $? "proper usage is --with-libm=STRING" "$LINENO" 5 fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default LIBM=\"$LIBM\"" >&5 -printf "%s\n" "default LIBM=\"$LIBM\"" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default LIBM=\"$LIBM\"" >&5 +printf "%s\n" "default LIBM=\"$LIBM\"" >&6; } ;; +esac fi @@ -24084,9 +25103,10 @@ then LIBC=$withval printf "%s\n" "set LIBC=\"$withval\"" >&6; } else as_fn_error $? "proper usage is --with-libc=STRING" "$LINENO" 5 fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default LIBC=\"$LIBC\"" >&5 -printf "%s\n" "default LIBC=\"$LIBC\"" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default LIBC=\"$LIBC\"" >&5 +printf "%s\n" "default LIBC=\"$LIBC\"" >&6; } ;; +esac fi @@ -24100,8 +25120,8 @@ printf %s "checking for x64 gcc inline assembler... " >&6; } if test ${ac_cv_gcc_asm_for_x64+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -24118,12 +25138,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_gcc_asm_for_x64=yes -else $as_nop - ac_cv_gcc_asm_for_x64=no +else case e in #( + e) ac_cv_gcc_asm_for_x64=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gcc_asm_for_x64" >&5 printf "%s\n" "$ac_cv_gcc_asm_for_x64" >&6; } @@ -24146,8 +25168,8 @@ printf %s "checking whether float word ordering is bigendian... " >&6; } if test ${ax_cv_c_float_words_bigendian+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_cv_c_float_words_bigendian=unknown cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -24184,7 +25206,8 @@ fi fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext + conftest$ac_exeext conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_c_float_words_bigendian" >&5 printf "%s\n" "$ax_cv_c_float_words_bigendian" >&6; } @@ -24232,8 +25255,8 @@ printf %s "checking whether we can use gcc inline assembler to get and set x87 c if test ${ac_cv_gcc_asm_for_x87+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -24252,12 +25275,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_gcc_asm_for_x87=yes -else $as_nop - ac_cv_gcc_asm_for_x87=no +else case e in #( + e) ac_cv_gcc_asm_for_x87=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gcc_asm_for_x87" >&5 printf "%s\n" "$ac_cv_gcc_asm_for_x87" >&6; } @@ -24275,8 +25300,8 @@ printf %s "checking whether we can use gcc inline assembler to get and set mc688 if test ${ac_cv_gcc_asm_for_mc68881+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -24295,12 +25320,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_gcc_asm_for_mc68881=yes -else $as_nop - ac_cv_gcc_asm_for_mc68881=no +else case e in #( + e) ac_cv_gcc_asm_for_mc68881=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gcc_asm_for_mc68881" >&5 printf "%s\n" "$ac_cv_gcc_asm_for_mc68881" >&6; } @@ -24323,16 +25350,16 @@ printf %s "checking for x87-style double rounding... " >&6; } if test ${ac_cv_x87_double_rounding+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) # $BASECFLAGS may affect the result ac_save_cc="$CC" CC="$CC $BASECFLAGS" if test "$cross_compiling" = yes then : ac_cv_x87_double_rounding=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -24358,15 +25385,18 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_x87_double_rounding=no -else $as_nop - ac_cv_x87_double_rounding=yes +else case e in #( + e) ac_cv_x87_double_rounding=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi CC="$ac_save_cc" - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_x87_double_rounding" >&5 printf "%s\n" "$ac_cv_x87_double_rounding" >&6; } @@ -24390,17 +25420,18 @@ LIBS="$LIBS $LIBM" for ac_func in acosh asinh atanh erf erfc expm1 log1p log2 do : - as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | $as_tr_sh` + as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | sed "$as_sed_sh"` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" if eval test \"x\$"$as_ac_var"\" = x"yes" then : cat >>confdefs.h <<_ACEOF -#define `printf "%s\n" "HAVE_$ac_func" | $as_tr_cpp` 1 +#define `printf "%s\n" "HAVE_$ac_func" | sed "$as_sed_cpp"` 1 _ACEOF -else $as_nop - as_fn_error $? "Python requires C99 compatible libm" "$LINENO" 5 - +else case e in #( + e) as_fn_error $? "Python requires C99 compatible libm" "$LINENO" 5 + ;; +esac fi done @@ -24411,12 +25442,12 @@ printf %s "checking whether POSIX semaphores are enabled... " >&6; } if test ${ac_cv_posix_semaphores_enabled+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_posix_semaphores_enabled=yes -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -24442,14 +25473,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_posix_semaphores_enabled=yes -else $as_nop - ac_cv_posix_semaphores_enabled=no +else case e in #( + e) ac_cv_posix_semaphores_enabled=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_posix_semaphores_enabled" >&5 printf "%s\n" "$ac_cv_posix_semaphores_enabled" >&6; } @@ -24467,12 +25501,12 @@ printf %s "checking for broken sem_getvalue... " >&6; } if test ${ac_cv_broken_sem_getvalue+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_broken_sem_getvalue=yes -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -24502,14 +25536,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_broken_sem_getvalue=no -else $as_nop - ac_cv_broken_sem_getvalue=yes +else case e in #( + e) ac_cv_broken_sem_getvalue=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_broken_sem_getvalue" >&5 printf "%s\n" "$ac_cv_broken_sem_getvalue" >&6; } @@ -24527,8 +25564,9 @@ ac_fn_check_decl "$LINENO" "RTLD_LAZY" "ac_cv_have_decl_RTLD_LAZY" "#include
>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_NOW" "ac_cv_have_decl_RTLD_NOW" "#include @@ -24536,8 +25574,9 @@ ac_fn_check_decl "$LINENO" "RTLD_NOW" "ac_cv_have_decl_RTLD_NOW" "#include >confdefs.h ac_fn_check_decl "$LINENO" "RTLD_GLOBAL" "ac_cv_have_decl_RTLD_GLOBAL" "#include @@ -24545,8 +25584,9 @@ ac_fn_check_decl "$LINENO" "RTLD_GLOBAL" "ac_cv_have_decl_RTLD_GLOBAL" "#include if test "x$ac_cv_have_decl_RTLD_GLOBAL" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_GLOBAL $ac_have_decl" >>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_LOCAL" "ac_cv_have_decl_RTLD_LOCAL" "#include @@ -24554,8 +25594,9 @@ ac_fn_check_decl "$LINENO" "RTLD_LOCAL" "ac_cv_have_decl_RTLD_LOCAL" "#include < if test "x$ac_cv_have_decl_RTLD_LOCAL" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_LOCAL $ac_have_decl" >>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_NODELETE" "ac_cv_have_decl_RTLD_NODELETE" "#include @@ -24563,8 +25604,9 @@ ac_fn_check_decl "$LINENO" "RTLD_NODELETE" "ac_cv_have_decl_RTLD_NODELETE" "#inc if test "x$ac_cv_have_decl_RTLD_NODELETE" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_NODELETE $ac_have_decl" >>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_NOLOAD" "ac_cv_have_decl_RTLD_NOLOAD" "#include @@ -24572,8 +25614,9 @@ ac_fn_check_decl "$LINENO" "RTLD_NOLOAD" "ac_cv_have_decl_RTLD_NOLOAD" "#include if test "x$ac_cv_have_decl_RTLD_NOLOAD" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_NOLOAD $ac_have_decl" >>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_DEEPBIND" "ac_cv_have_decl_RTLD_DEEPBIND" "#include @@ -24581,8 +25624,9 @@ ac_fn_check_decl "$LINENO" "RTLD_DEEPBIND" "ac_cv_have_decl_RTLD_DEEPBIND" "#inc if test "x$ac_cv_have_decl_RTLD_DEEPBIND" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_DEEPBIND $ac_have_decl" >>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_MEMBER" "ac_cv_have_decl_RTLD_MEMBER" "#include @@ -24590,8 +25634,9 @@ ac_fn_check_decl "$LINENO" "RTLD_MEMBER" "ac_cv_have_decl_RTLD_MEMBER" "#include if test "x$ac_cv_have_decl_RTLD_MEMBER" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_MEMBER $ac_have_decl" >>confdefs.h @@ -24618,9 +25663,10 @@ printf "%s\n" "$enable_big_digits" >&6; } printf "%s\n" "#define PYLONG_BITS_IN_DIGIT $enable_big_digits" >>confdefs.h -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no value specified" >&5 -printf "%s\n" "no value specified" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no value specified" >&5 +printf "%s\n" "no value specified" >&6; } ;; +esac fi @@ -24634,9 +25680,10 @@ printf "%s\n" "#define HAVE_WCHAR_H 1" >>confdefs.h wchar_h="yes" -else $as_nop - wchar_h="no" - +else case e in #( + e) wchar_h="no" + ;; +esac fi @@ -24645,29 +25692,31 @@ if test "$wchar_h" = yes then # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of wchar_t" >&5 printf %s "checking size of wchar_t... " >&6; } if test ${ac_cv_sizeof_wchar_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (wchar_t))" "ac_cv_sizeof_wchar_t" "#include +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (wchar_t))" "ac_cv_sizeof_wchar_t" "#include " then : -else $as_nop - if test "$ac_cv_type_wchar_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_wchar_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (wchar_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_wchar_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_wchar_t" >&5 printf "%s\n" "$ac_cv_sizeof_wchar_t" >&6; } @@ -24688,13 +25737,13 @@ printf %s "checking whether wchar_t is signed... " >&6; } if test ${ac_cv_wchar_t_signed+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_wchar_t_signed=yes -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -24708,13 +25757,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_wchar_t_signed=yes -else $as_nop - ac_cv_wchar_t_signed=no +else case e in #( + e) ac_cv_wchar_t_signed=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_wchar_t_signed" >&5 printf "%s\n" "$ac_cv_wchar_t_signed" >&6; } @@ -24758,8 +25810,8 @@ printf %s "checking whether byte ordering is bigendian... " >&6; } if test ${ac_cv_c_bigendian+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_c_bigendian=unknown +else case e in #( + e) ac_cv_c_bigendian=unknown # See if we're dealing with a universal compiler. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -24805,8 +25857,8 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext int main (void) { -#if ! (defined BYTE_ORDER && defined BIG_ENDIAN \ - && defined LITTLE_ENDIAN && BYTE_ORDER && BIG_ENDIAN \ +#if ! (defined BYTE_ORDER && defined BIG_ENDIAN \\ + && defined LITTLE_ENDIAN && BYTE_ORDER && BIG_ENDIAN \\ && LITTLE_ENDIAN) bogus endian macros #endif @@ -24837,8 +25889,9 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_c_bigendian=yes -else $as_nop - ac_cv_c_bigendian=no +else case e in #( + e) ac_cv_c_bigendian=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext fi @@ -24882,8 +25935,9 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_c_bigendian=yes -else $as_nop - ac_cv_c_bigendian=no +else case e in #( + e) ac_cv_c_bigendian=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext fi @@ -24910,22 +25964,23 @@ unsigned short int ascii_mm[] = int use_ebcdic (int i) { return ebcdic_mm[i] + ebcdic_ii[i]; } - extern int foo; - -int -main (void) -{ -return use_ascii (foo) == use_ebcdic (foo); - ; - return 0; -} + int + main (int argc, char **argv) + { + /* Intimidate the compiler so that it does not + optimize the arrays away. */ + char *p = argv[0]; + ascii_mm[1] = *p++; ebcdic_mm[1] = *p++; + ascii_ii[1] = *p++; ebcdic_ii[1] = *p++; + return use_ascii (argc) == use_ebcdic (*p); + } _ACEOF -if ac_fn_c_try_compile "$LINENO" +if ac_fn_c_try_link "$LINENO" then : - if grep BIGenDianSyS conftest.$ac_objext >/dev/null; then + if grep BIGenDianSyS conftest$ac_exeext >/dev/null; then ac_cv_c_bigendian=yes fi - if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then + if grep LiTTleEnDian conftest$ac_exeext >/dev/null ; then if test "$ac_cv_c_bigendian" = unknown; then ac_cv_c_bigendian=no else @@ -24934,9 +25989,10 @@ then : fi fi fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $ac_includes_default int @@ -24959,14 +26015,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_c_bigendian=no -else $as_nop - ac_cv_c_bigendian=yes +else case e in #( + e) ac_cv_c_bigendian=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - fi + fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_bigendian" >&5 printf "%s\n" "$ac_cv_c_bigendian" >&6; } @@ -25084,9 +26143,10 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -25117,9 +26177,10 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -25130,13 +26191,13 @@ printf %s "checking whether right shift extends the sign bit... " >&6; } if test ${ac_cv_rshift_extends_sign+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_rshift_extends_sign=yes -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main(void) @@ -25148,13 +26209,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_rshift_extends_sign=yes -else $as_nop - ac_cv_rshift_extends_sign=no +else case e in #( + e) ac_cv_rshift_extends_sign=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_rshift_extends_sign" >&5 printf "%s\n" "$ac_cv_rshift_extends_sign" >&6; } @@ -25171,8 +26235,8 @@ printf %s "checking for getc_unlocked() and friends... " >&6; } if test ${ac_cv_have_getc_unlocked+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -25192,11 +26256,13 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_have_getc_unlocked=yes -else $as_nop - ac_cv_have_getc_unlocked=no +else case e in #( + e) ac_cv_have_getc_unlocked=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext + conftest$ac_exeext conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_getc_unlocked" >&5 printf "%s\n" "$ac_cv_have_getc_unlocked" >&6; } @@ -25227,9 +26293,10 @@ then : ;; esac -else $as_nop - with_readline=readline - +else case e in #( + e) with_readline=readline + ;; +esac fi @@ -25316,16 +26383,22 @@ printf %s "checking for readline in -lreadline... " >&6; } if test ${ac_cv_lib_readline_readline+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lreadline $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char readline (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char readline (void); int main (void) { @@ -25337,12 +26410,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_readline_readline=yes -else $as_nop - ac_cv_lib_readline_readline=no +else case e in #( + e) ac_cv_lib_readline_readline=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_readline_readline" >&5 printf "%s\n" "$ac_cv_lib_readline_readline" >&6; } @@ -25353,13 +26428,15 @@ then : READLINE_CFLAGS=${LIBREADLINE_CFLAGS-""} READLINE_LIBS=${LIBREADLINE_LIBS-"-lreadline"} -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi done @@ -25395,16 +26472,22 @@ printf %s "checking for readline in -lreadline... " >&6; } if test ${ac_cv_lib_readline_readline+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lreadline $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char readline (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char readline (void); int main (void) { @@ -25416,12 +26499,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_readline_readline=yes -else $as_nop - ac_cv_lib_readline_readline=no +else case e in #( + e) ac_cv_lib_readline_readline=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_readline_readline" >&5 printf "%s\n" "$ac_cv_lib_readline_readline" >&6; } @@ -25432,13 +26517,15 @@ then : READLINE_CFLAGS=${LIBREADLINE_CFLAGS-""} READLINE_LIBS=${LIBREADLINE_LIBS-"-lreadline"} -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi done @@ -25547,16 +26634,22 @@ printf %s "checking for readline in -ledit... " >&6; } if test ${ac_cv_lib_edit_readline+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-ledit $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char readline (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char readline (void); int main (void) { @@ -25568,12 +26661,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_edit_readline=yes -else $as_nop - ac_cv_lib_edit_readline=no +else case e in #( + e) ac_cv_lib_edit_readline=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_edit_readline" >&5 printf "%s\n" "$ac_cv_lib_edit_readline" >&6; } @@ -25586,13 +26681,15 @@ then : READLINE_CFLAGS=${LIBEDIT_CFLAGS-""} READLINE_LIBS=${LIBEDIT_LIBS-"-ledit"} -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi done @@ -25628,16 +26725,22 @@ printf %s "checking for readline in -ledit... " >&6; } if test ${ac_cv_lib_edit_readline+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-ledit $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char readline (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char readline (void); int main (void) { @@ -25649,12 +26752,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_edit_readline=yes -else $as_nop - ac_cv_lib_edit_readline=no +else case e in #( + e) ac_cv_lib_edit_readline=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_edit_readline" >&5 printf "%s\n" "$ac_cv_lib_edit_readline" >&6; } @@ -25667,13 +26772,15 @@ then : READLINE_CFLAGS=${LIBEDIT_CFLAGS-""} READLINE_LIBS=${LIBEDIT_LIBS-"-ledit"} -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi done @@ -25711,8 +26818,8 @@ then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_readline (CFLAGS: $READLINE_CFLAGS, LIBS: $READLINE_LIBS)" >&5 printf "%s\n" "$with_readline (CFLAGS: $READLINE_CFLAGS, LIBS: $READLINE_LIBS)" >&6; } @@ -25773,8 +26880,8 @@ printf %s "checking for rl_pre_input_hook in -l$LIBREADLINE... " >&6; } if test ${ac_cv_readline_rl_pre_input_hook+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -25797,13 +26904,15 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_readline_rl_pre_input_hook=yes -else $as_nop - ac_cv_readline_rl_pre_input_hook=no - +else case e in #( + e) ac_cv_readline_rl_pre_input_hook=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_pre_input_hook" >&5 printf "%s\n" "$ac_cv_readline_rl_pre_input_hook" >&6; } @@ -25822,8 +26931,8 @@ printf %s "checking for rl_completion_display_matches_hook in -l$LIBREADLINE... if test ${ac_cv_readline_rl_completion_display_matches_hook+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -25846,13 +26955,15 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_readline_rl_completion_display_matches_hook=yes -else $as_nop - ac_cv_readline_rl_completion_display_matches_hook=no - +else case e in #( + e) ac_cv_readline_rl_completion_display_matches_hook=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_completion_display_matches_hook" >&5 printf "%s\n" "$ac_cv_readline_rl_completion_display_matches_hook" >&6; } @@ -25871,8 +26982,8 @@ printf %s "checking for rl_resize_terminal in -l$LIBREADLINE... " >&6; } if test ${ac_cv_readline_rl_resize_terminal+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -25895,13 +27006,15 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_readline_rl_resize_terminal=yes -else $as_nop - ac_cv_readline_rl_resize_terminal=no - +else case e in #( + e) ac_cv_readline_rl_resize_terminal=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_resize_terminal" >&5 printf "%s\n" "$ac_cv_readline_rl_resize_terminal" >&6; } @@ -25920,8 +27033,8 @@ printf %s "checking for rl_completion_matches in -l$LIBREADLINE... " >&6; } if test ${ac_cv_readline_rl_completion_matches+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -25944,13 +27057,15 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_readline_rl_completion_matches=yes -else $as_nop - ac_cv_readline_rl_completion_matches=no - +else case e in #( + e) ac_cv_readline_rl_completion_matches=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_completion_matches" >&5 printf "%s\n" "$ac_cv_readline_rl_completion_matches" >&6; } @@ -25988,8 +27103,8 @@ printf %s "checking for append_history in -l$LIBREADLINE... " >&6; } if test ${ac_cv_readline_append_history+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -26012,13 +27127,15 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_readline_append_history=yes -else $as_nop - ac_cv_readline_append_history=no - +else case e in #( + e) ac_cv_readline_append_history=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_append_history" >&5 printf "%s\n" "$ac_cv_readline_append_history" >&6; } @@ -26058,8 +27175,8 @@ printf %s "checking if rl_startup_hook takes arguments... " >&6; } if test ${ac_cv_readline_rl_startup_hook_takes_args+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -26083,12 +27200,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_readline_rl_startup_hook_takes_args=yes -else $as_nop - ac_cv_readline_rl_startup_hook_takes_args=no - +else case e in #( + e) ac_cv_readline_rl_startup_hook_takes_args=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_startup_hook_takes_args" >&5 printf "%s\n" "$ac_cv_readline_rl_startup_hook_takes_args" >&6; } @@ -26108,7 +27227,8 @@ CPPFLAGS=$save_CPPFLAGS LDFLAGS=$save_LDFLAGS LIBS=$save_LIBS - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for broken nice()" >&5 @@ -26116,13 +27236,13 @@ printf %s "checking for broken nice()... " >&6; } if test ${ac_cv_broken_nice+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_broken_nice=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -26139,13 +27259,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_broken_nice=yes -else $as_nop - ac_cv_broken_nice=no +else case e in #( + e) ac_cv_broken_nice=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_broken_nice" >&5 printf "%s\n" "$ac_cv_broken_nice" >&6; } @@ -26161,12 +27284,12 @@ printf %s "checking for broken poll()... " >&6; } if test ${ac_cv_broken_poll+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_broken_poll=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -26192,13 +27315,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_broken_poll=yes -else $as_nop - ac_cv_broken_poll=no +else case e in #( + e) ac_cv_broken_poll=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_broken_poll" >&5 printf "%s\n" "$ac_cv_broken_poll" >&6; } @@ -26215,13 +27341,13 @@ printf %s "checking for working tzset()... " >&6; } if test ${ac_cv_working_tzset+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_working_tzset=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -26291,13 +27417,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_working_tzset=yes -else $as_nop - ac_cv_working_tzset=no +else case e in #( + e) ac_cv_working_tzset=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_tzset" >&5 printf "%s\n" "$ac_cv_working_tzset" >&6; } @@ -26314,8 +27443,8 @@ printf %s "checking for tv_nsec in struct stat... " >&6; } if test ${ac_cv_stat_tv_nsec+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -26332,10 +27461,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_stat_tv_nsec=yes -else $as_nop - ac_cv_stat_tv_nsec=no +else case e in #( + e) ac_cv_stat_tv_nsec=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_stat_tv_nsec" >&5 printf "%s\n" "$ac_cv_stat_tv_nsec" >&6; } @@ -26352,8 +27483,8 @@ printf %s "checking for tv_nsec2 in struct stat... " >&6; } if test ${ac_cv_stat_tv_nsec2+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -26370,10 +27501,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_stat_tv_nsec2=yes -else $as_nop - ac_cv_stat_tv_nsec2=no +else case e in #( + e) ac_cv_stat_tv_nsec2=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_stat_tv_nsec2" >&5 printf "%s\n" "$ac_cv_stat_tv_nsec2" >&6; } @@ -26389,13 +27522,13 @@ printf %s "checking whether year with century should be normalized for strftime. if test ${ac_cv_normalize_century+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_normalize_century=yes -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -26419,13 +27552,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_normalize_century=yes -else $as_nop - ac_cv_normalize_century=no +else case e in #( + e) ac_cv_normalize_century=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_normalize_century" >&5 printf "%s\n" "$ac_cv_normalize_century" >&6; } @@ -26441,13 +27577,13 @@ printf %s "checking whether C99-compatible strftime specifiers are supported... if test ${ac_cv_strftime_c99_support+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_strftime_c99_support= -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -26471,13 +27607,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_strftime_c99_support=yes -else $as_nop - as_fn_error $? "Python requires C99-compatible strftime specifiers" "$LINENO" 5 +else case e in #( + e) as_fn_error $? "Python requires C99-compatible strftime specifiers" "$LINENO" 5 ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_strftime_c99_support" >&5 printf "%s\n" "$ac_cv_strftime_c99_support" >&6; } @@ -26868,15 +28007,21 @@ printf %s "checking for library containing initscr... " >&6; } if test ${ac_cv_search_initscr+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char initscr (void); int main (void) { @@ -26907,11 +28052,13 @@ done if test ${ac_cv_search_initscr+y} then : -else $as_nop - ac_cv_search_initscr=no +else case e in #( + e) ac_cv_search_initscr=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_initscr" >&5 printf "%s\n" "$ac_cv_search_initscr" >&6; } @@ -26924,8 +28071,9 @@ then : have_curses=yes CURSES_LIBS=${CURSES_LIBS-"$ac_cv_search_initscr"} fi -else $as_nop - have_curses=no +else case e in #( + e) have_curses=no ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing update_panels" >&5 @@ -26933,15 +28081,21 @@ printf %s "checking for library containing update_panels... " >&6; } if test ${ac_cv_search_update_panels+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char update_panels (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char update_panels (void); int main (void) { @@ -26972,11 +28126,13 @@ done if test ${ac_cv_search_update_panels+y} then : -else $as_nop - ac_cv_search_update_panels=no +else case e in #( + e) ac_cv_search_update_panels=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_update_panels" >&5 printf "%s\n" "$ac_cv_search_update_panels" >&6; } @@ -26989,8 +28145,9 @@ then : have_panel=yes PANEL_LIBS=${PANEL_LIBS-"$ac_cv_search_update_panels"} fi -else $as_nop - have_panel=no +else case e in #( + e) have_panel=no ;; +esac fi @@ -27042,8 +28199,8 @@ printf %s "checking whether mvwdelch is an expression... " >&6; } if test ${ac_cv_mvwdelch_is_expression+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27075,10 +28232,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_mvwdelch_is_expression=yes -else $as_nop - ac_cv_mvwdelch_is_expression=no +else case e in #( + e) ac_cv_mvwdelch_is_expression=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_mvwdelch_is_expression" >&5 printf "%s\n" "$ac_cv_mvwdelch_is_expression" >&6; } @@ -27095,8 +28254,8 @@ printf %s "checking whether WINDOW has _flags... " >&6; } if test ${ac_cv_window_has_flags+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27128,10 +28287,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_window_has_flags=yes -else $as_nop - ac_cv_window_has_flags=no +else case e in #( + e) ac_cv_window_has_flags=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_window_has_flags" >&5 printf "%s\n" "$ac_cv_window_has_flags" >&6; } @@ -27153,8 +28314,8 @@ printf %s "checking for curses function is_pad... " >&6; } if test ${ac_cv_lib_curses_is_pad+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27187,11 +28348,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_is_pad=yes -else $as_nop - ac_cv_lib_curses_is_pad=no +else case e in #( + e) ac_cv_lib_curses_is_pad=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_pad" >&5 printf "%s\n" "$ac_cv_lib_curses_is_pad" >&6; } @@ -27211,8 +28374,8 @@ printf %s "checking for curses function is_term_resized... " >&6; } if test ${ac_cv_lib_curses_is_term_resized+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27245,11 +28408,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_is_term_resized=yes -else $as_nop - ac_cv_lib_curses_is_term_resized=no +else case e in #( + e) ac_cv_lib_curses_is_term_resized=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_term_resized" >&5 printf "%s\n" "$ac_cv_lib_curses_is_term_resized" >&6; } @@ -27269,8 +28434,8 @@ printf %s "checking for curses function resize_term... " >&6; } if test ${ac_cv_lib_curses_resize_term+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27303,11 +28468,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_resize_term=yes -else $as_nop - ac_cv_lib_curses_resize_term=no +else case e in #( + e) ac_cv_lib_curses_resize_term=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_resize_term" >&5 printf "%s\n" "$ac_cv_lib_curses_resize_term" >&6; } @@ -27327,8 +28494,8 @@ printf %s "checking for curses function resizeterm... " >&6; } if test ${ac_cv_lib_curses_resizeterm+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27361,11 +28528,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_resizeterm=yes -else $as_nop - ac_cv_lib_curses_resizeterm=no +else case e in #( + e) ac_cv_lib_curses_resizeterm=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_resizeterm" >&5 printf "%s\n" "$ac_cv_lib_curses_resizeterm" >&6; } @@ -27385,8 +28554,8 @@ printf %s "checking for curses function immedok... " >&6; } if test ${ac_cv_lib_curses_immedok+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27419,11 +28588,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_immedok=yes -else $as_nop - ac_cv_lib_curses_immedok=no +else case e in #( + e) ac_cv_lib_curses_immedok=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_immedok" >&5 printf "%s\n" "$ac_cv_lib_curses_immedok" >&6; } @@ -27443,8 +28614,8 @@ printf %s "checking for curses function syncok... " >&6; } if test ${ac_cv_lib_curses_syncok+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27477,11 +28648,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_syncok=yes -else $as_nop - ac_cv_lib_curses_syncok=no +else case e in #( + e) ac_cv_lib_curses_syncok=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_syncok" >&5 printf "%s\n" "$ac_cv_lib_curses_syncok" >&6; } @@ -27501,8 +28674,8 @@ printf %s "checking for curses function wchgat... " >&6; } if test ${ac_cv_lib_curses_wchgat+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27535,11 +28708,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_wchgat=yes -else $as_nop - ac_cv_lib_curses_wchgat=no +else case e in #( + e) ac_cv_lib_curses_wchgat=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wchgat" >&5 printf "%s\n" "$ac_cv_lib_curses_wchgat" >&6; } @@ -27559,8 +28734,8 @@ printf %s "checking for curses function filter... " >&6; } if test ${ac_cv_lib_curses_filter+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27593,11 +28768,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_filter=yes -else $as_nop - ac_cv_lib_curses_filter=no +else case e in #( + e) ac_cv_lib_curses_filter=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_filter" >&5 printf "%s\n" "$ac_cv_lib_curses_filter" >&6; } @@ -27617,8 +28794,8 @@ printf %s "checking for curses function has_key... " >&6; } if test ${ac_cv_lib_curses_has_key+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27651,11 +28828,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_has_key=yes -else $as_nop - ac_cv_lib_curses_has_key=no +else case e in #( + e) ac_cv_lib_curses_has_key=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_key" >&5 printf "%s\n" "$ac_cv_lib_curses_has_key" >&6; } @@ -27675,8 +28854,8 @@ printf %s "checking for curses function typeahead... " >&6; } if test ${ac_cv_lib_curses_typeahead+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27709,11 +28888,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_typeahead=yes -else $as_nop - ac_cv_lib_curses_typeahead=no +else case e in #( + e) ac_cv_lib_curses_typeahead=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_typeahead" >&5 printf "%s\n" "$ac_cv_lib_curses_typeahead" >&6; } @@ -27733,8 +28914,8 @@ printf %s "checking for curses function use_env... " >&6; } if test ${ac_cv_lib_curses_use_env+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define NCURSES_OPAQUE 0 @@ -27767,11 +28948,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_use_env=yes -else $as_nop - ac_cv_lib_curses_use_env=no +else case e in #( + e) ac_cv_lib_curses_use_env=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_env" >&5 printf "%s\n" "$ac_cv_lib_curses_use_env" >&6; } @@ -27822,14 +29005,15 @@ printf %s "checking for /dev/ptmx... " >&6; } if test ${ac_cv_file__dev_ptmx+y} then : printf %s "(cached) " >&6 -else $as_nop - test "$cross_compiling" = yes && +else case e in #( + e) test "$cross_compiling" = yes && as_fn_error $? "cannot check for file existence when cross compiling" "$LINENO" 5 if test -r "/dev/ptmx"; then ac_cv_file__dev_ptmx=yes else ac_cv_file__dev_ptmx=no -fi +fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_file__dev_ptmx" >&5 printf "%s\n" "$ac_cv_file__dev_ptmx" >&6; } @@ -27848,14 +29032,15 @@ printf %s "checking for /dev/ptc... " >&6; } if test ${ac_cv_file__dev_ptc+y} then : printf %s "(cached) " >&6 -else $as_nop - test "$cross_compiling" = yes && +else case e in #( + e) test "$cross_compiling" = yes && as_fn_error $? "cannot check for file existence when cross compiling" "$LINENO" 5 if test -r "/dev/ptc"; then ac_cv_file__dev_ptc=yes else ac_cv_file__dev_ptc=no -fi +fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_file__dev_ptc" >&5 printf "%s\n" "$ac_cv_file__dev_ptc" >&6; } @@ -27891,10 +29076,11 @@ then : printf "%s\n" "#define HAVE_SOCKLEN_T 1" >>confdefs.h -else $as_nop - +else case e in #( + e) printf "%s\n" "#define socklen_t int" >>confdefs.h - + ;; +esac fi @@ -27903,12 +29089,12 @@ printf %s "checking for broken mbstowcs... " >&6; } if test ${ac_cv_broken_mbstowcs+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_broken_mbstowcs=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -27925,13 +29111,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_broken_mbstowcs=no -else $as_nop - ac_cv_broken_mbstowcs=yes +else case e in #( + e) ac_cv_broken_mbstowcs=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_broken_mbstowcs" >&5 printf "%s\n" "$ac_cv_broken_mbstowcs" >&6; } @@ -27967,9 +29156,10 @@ printf "%s\n" "#define USE_COMPUTED_GOTOS 0" >>confdefs.h printf "%s\n" "no" >&6; } fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no value specified" >&5 -printf "%s\n" "no value specified" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no value specified" >&5 +printf "%s\n" "no value specified" >&6; } ;; +esac fi @@ -27978,16 +29168,16 @@ printf %s "checking whether $CC supports computed gotos... " >&6; } if test ${ac_cv_computed_gotos+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : if test "${with_computed_gotos+set}" = set; then ac_cv_computed_gotos="$with_computed_gotos -- configured --with(out)-computed-gotos" else ac_cv_computed_gotos=no fi -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main(int argc, char **argv) @@ -28005,13 +29195,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_computed_gotos=yes -else $as_nop - ac_cv_computed_gotos=no +else case e in #( + e) ac_cv_computed_gotos=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_computed_gotos" >&5 printf "%s\n" "$ac_cv_computed_gotos" >&6; } @@ -28078,8 +29271,8 @@ printf %s "checking for -O2... " >&6; } if test ${ac_cv_compile_o2+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) saved_cflags="$CFLAGS" CFLAGS="-O2" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -28096,12 +29289,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_compile_o2=yes -else $as_nop - ac_cv_compile_o2=no +else case e in #( + e) ac_cv_compile_o2=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS="$saved_cflags" - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_compile_o2" >&5 printf "%s\n" "$ac_cv_compile_o2" >&6; } @@ -28118,8 +29313,8 @@ fi if test "$cross_compiling" = yes then : have_glibc_memmove_bug=undefined -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -28141,11 +29336,13 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : have_glibc_memmove_bug=no -else $as_nop - have_glibc_memmove_bug=yes +else case e in #( + e) have_glibc_memmove_bug=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi CFLAGS="$saved_cflags" @@ -28170,8 +29367,8 @@ printf %s "checking for gcc ipa-pure-const bug... " >&6; } if test "$cross_compiling" = yes then : have_ipa_pure_const_bug=undefined -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ __attribute__((noinline)) int @@ -28194,11 +29391,13 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : have_ipa_pure_const_bug=no -else $as_nop - have_ipa_pure_const_bug=yes +else case e in #( + e) have_ipa_pure_const_bug=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi CFLAGS="$saved_cflags" @@ -28221,8 +29420,8 @@ printf %s "checking for ensurepip... " >&6; } if test ${with_ensurepip+y} then : withval=$with_ensurepip; -else $as_nop - +else case e in #( + e) case $ac_sys_system in #( Emscripten) : with_ensurepip=no ;; #( @@ -28234,7 +29433,8 @@ else $as_nop with_ensurepip=upgrade ;; esac - + ;; +esac fi case $with_ensurepip in #( @@ -28257,8 +29457,8 @@ printf %s "checking if the dirent structure of a d_type field... " >&6; } if test ${ac_cv_dirent_d_type+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -28275,12 +29475,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_dirent_d_type=yes -else $as_nop - ac_cv_dirent_d_type=no +else case e in #( + e) ac_cv_dirent_d_type=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_dirent_d_type" >&5 printf "%s\n" "$ac_cv_dirent_d_type" >&6; } @@ -28300,8 +29502,8 @@ printf %s "checking for the Linux getrandom() syscall... " >&6; } if test ${ac_cv_getrandom_syscall+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -28325,12 +29527,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_getrandom_syscall=yes -else $as_nop - ac_cv_getrandom_syscall=no +else case e in #( + e) ac_cv_getrandom_syscall=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_getrandom_syscall" >&5 printf "%s\n" "$ac_cv_getrandom_syscall" >&6; } @@ -28351,8 +29555,8 @@ printf %s "checking for the getrandom() function... " >&6; } if test ${ac_cv_func_getrandom+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -28374,12 +29578,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_func_getrandom=yes -else $as_nop - ac_cv_func_getrandom=no +else case e in #( + e) ac_cv_func_getrandom=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getrandom" >&5 printf "%s\n" "$ac_cv_func_getrandom" >&6; } @@ -28407,15 +29613,21 @@ printf %s "checking for library containing shm_open... " >&6; } if test ${ac_cv_search_shm_open+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char shm_open (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char shm_open (void); int main (void) { @@ -28446,11 +29658,13 @@ done if test ${ac_cv_search_shm_open+y} then : -else $as_nop - ac_cv_search_shm_open=no +else case e in #( + e) ac_cv_search_shm_open=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_shm_open" >&5 printf "%s\n" "$ac_cv_search_shm_open" >&6; } @@ -28478,16 +29692,17 @@ fi for ac_func in shm_open shm_unlink do : - as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | $as_tr_sh` + as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | sed "$as_sed_sh"` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" if eval test \"x\$"$as_ac_var"\" = x"yes" then : cat >>confdefs.h <<_ACEOF -#define `printf "%s\n" "HAVE_$ac_func" | $as_tr_cpp` 1 +#define `printf "%s\n" "HAVE_$ac_func" | sed "$as_sed_cpp"` 1 _ACEOF have_posix_shmem=yes -else $as_nop - have_posix_shmem=no +else case e in #( + e) have_posix_shmem=no ;; +esac fi done @@ -28516,8 +29731,8 @@ then : ;; esac -else $as_nop - +else case e in #( + e) # if pkg-config is installed and openssl has installed a .pc file, # then use that information and don't search ssldirs if test -n "$ac_tool_prefix"; then @@ -28528,8 +29743,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_PKG_CONFIG+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$PKG_CONFIG"; then +else case e in #( + e) if test -n "$PKG_CONFIG"; then ac_cv_prog_PKG_CONFIG="$PKG_CONFIG" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -28551,7 +29766,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi PKG_CONFIG=$ac_cv_prog_PKG_CONFIG if test -n "$PKG_CONFIG"; then @@ -28573,8 +29789,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_PKG_CONFIG+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_PKG_CONFIG"; then +else case e in #( + e) if test -n "$ac_ct_PKG_CONFIG"; then ac_cv_prog_ac_ct_PKG_CONFIG="$ac_ct_PKG_CONFIG" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -28596,7 +29812,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_PKG_CONFIG=$ac_cv_prog_ac_ct_PKG_CONFIG if test -n "$ac_ct_PKG_CONFIG"; then @@ -28636,7 +29853,8 @@ fi ssldirs="/usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr" fi - + ;; +esac fi @@ -28699,12 +29917,13 @@ then : printf "%s\n" "yes" >&6; } have_openssl=yes -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } have_openssl=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -28723,15 +29942,16 @@ then : rpath_arg="-Wl,--enable-new-dtags,-rpath=" -else $as_nop - +else case e in #( + e) if test "$ac_sys_system" = "Darwin" then rpath_arg="-Wl,-rpath," else rpath_arg="-Wl,-rpath=" fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-openssl-rpath" >&5 @@ -28741,9 +29961,10 @@ printf %s "checking for --with-openssl-rpath... " >&6; } if test ${with_openssl_rpath+y} then : withval=$with_openssl_rpath; -else $as_nop - with_openssl_rpath=no - +else case e in #( + e) with_openssl_rpath=no + ;; +esac fi case $with_openssl_rpath in #( @@ -28769,8 +29990,9 @@ then : OPENSSL_RPATH="$with_openssl_rpath" OPENSSL_LDFLAGS_RPATH="${rpath_arg}$with_openssl_rpath" -else $as_nop - as_fn_error $? "--with-openssl-rpath \"$with_openssl_rpath\" is not a directory" "$LINENO" 5 +else case e in #( + e) as_fn_error $? "--with-openssl-rpath \"$with_openssl_rpath\" is not a directory" "$LINENO" 5 ;; +esac fi ;; @@ -28833,8 +30055,8 @@ printf %s "checking whether OpenSSL provides required ssl module APIs... " >&6; if test ${ac_cv_working_openssl_ssl+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -28864,12 +30086,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_working_openssl_ssl=yes -else $as_nop - ac_cv_working_openssl_ssl=no +else case e in #( + e) ac_cv_working_openssl_ssl=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_openssl_ssl" >&5 printf "%s\n" "$ac_cv_working_openssl_ssl" >&6; } @@ -28896,8 +30120,8 @@ printf %s "checking whether OpenSSL provides required hashlib module APIs... " > if test ${ac_cv_working_openssl_hashlib+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -28924,12 +30148,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_working_openssl_hashlib=yes -else $as_nop - ac_cv_working_openssl_hashlib=no +else case e in #( + e) ac_cv_working_openssl_hashlib=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_openssl_hashlib" >&5 printf "%s\n" "$ac_cv_working_openssl_hashlib" >&6; } @@ -28971,13 +30197,14 @@ case "$withval" in ;; esac -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: python" >&5 printf "%s\n" "python" >&6; } printf "%s\n" "#define PY_SSL_DEFAULT_CIPHERS 1" >>confdefs.h - + ;; +esac fi @@ -29000,8 +30227,9 @@ then : ;; esac -else $as_nop - with_builtin_hashlib_hashes=$default_hashlib_hashes +else case e in #( + e) with_builtin_hashlib_hashes=$default_hashlib_hashes ;; +esac fi @@ -29043,12 +30271,14 @@ then : if test "x$enable_test_modules" = xyes then : TEST_MODULES=yes -else $as_nop - TEST_MODULES=no +else case e in #( + e) TEST_MODULES=no ;; +esac fi -else $as_nop - TEST_MODULES=yes +else case e in #( + e) TEST_MODULES=yes ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $TEST_MODULES" >&5 @@ -29077,8 +30307,8 @@ printf %s "checking whether libatomic is needed by ... " >&6; } if test ${ac_cv_libatomic_needed+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ // pyatomic.h needs uint64_t and Py_ssize_t types @@ -29121,11 +30351,13 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_libatomic_needed=no -else $as_nop - ac_cv_libatomic_needed=yes +else case e in #( + e) ac_cv_libatomic_needed=yes ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext + conftest$ac_exeext conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_libatomic_needed" >&5 printf "%s\n" "$ac_cv_libatomic_needed" >&6; } @@ -29778,11 +31010,13 @@ then : if test "$ac_cv_func_sem_unlink" = "yes" then : py_cv_module__multiprocessing=yes -else $as_nop - py_cv_module__multiprocessing=missing +else case e in #( + e) py_cv_module__multiprocessing=missing ;; +esac fi -else $as_nop - py_cv_module__multiprocessing=disabled +else case e in #( + e) py_cv_module__multiprocessing=disabled ;; +esac fi fi @@ -29816,11 +31050,13 @@ then : if test "$have_posix_shmem" = "yes" then : py_cv_module__posixshmem=yes -else $as_nop - py_cv_module__posixshmem=missing +else case e in #( + e) py_cv_module__posixshmem=missing ;; +esac fi -else $as_nop - py_cv_module__posixshmem=disabled +else case e in #( + e) py_cv_module__posixshmem=disabled ;; +esac fi fi @@ -29945,11 +31181,13 @@ then : if test "$ac_cv_header_sys_ioctl_h" = "yes" -a "$ac_cv_header_fcntl_h" = "yes" then : py_cv_module_fcntl=yes -else $as_nop - py_cv_module_fcntl=missing +else case e in #( + e) py_cv_module_fcntl=missing ;; +esac fi -else $as_nop - py_cv_module_fcntl=disabled +else case e in #( + e) py_cv_module_fcntl=disabled ;; +esac fi fi @@ -29983,11 +31221,13 @@ then : if test "$ac_cv_header_sys_mman_h" = "yes" -a "$ac_cv_header_sys_stat_h" = "yes" then : py_cv_module_mmap=yes -else $as_nop - py_cv_module_mmap=missing +else case e in #( + e) py_cv_module_mmap=missing ;; +esac fi -else $as_nop - py_cv_module_mmap=disabled +else case e in #( + e) py_cv_module_mmap=disabled ;; +esac fi fi @@ -30021,11 +31261,13 @@ then : if test "$ac_cv_header_sys_socket_h" = "yes" -a "$ac_cv_header_sys_types_h" = "yes" -a "$ac_cv_header_netinet_in_h" = "yes" then : py_cv_module__socket=yes -else $as_nop - py_cv_module__socket=missing +else case e in #( + e) py_cv_module__socket=missing ;; +esac fi -else $as_nop - py_cv_module__socket=disabled +else case e in #( + e) py_cv_module__socket=disabled ;; +esac fi fi @@ -30061,11 +31303,13 @@ then : { test "$ac_cv_func_getgrgid" = "yes" || test "$ac_cv_func_getgrgid_r" = "yes"; } then : py_cv_module_grp=yes -else $as_nop - py_cv_module_grp=missing +else case e in #( + e) py_cv_module_grp=missing ;; +esac fi -else $as_nop - py_cv_module_grp=disabled +else case e in #( + e) py_cv_module_grp=disabled ;; +esac fi fi @@ -30099,11 +31343,13 @@ then : if test "$ac_cv_func_getpwuid" = yes -o "$ac_cv_func_getpwuid_r" = yes then : py_cv_module_pwd=yes -else $as_nop - py_cv_module_pwd=missing +else case e in #( + e) py_cv_module_pwd=missing ;; +esac fi -else $as_nop - py_cv_module_pwd=disabled +else case e in #( + e) py_cv_module_pwd=disabled ;; +esac fi fi @@ -30137,11 +31383,13 @@ then : if test "$ac_cv_header_sys_resource_h" = yes then : py_cv_module_resource=yes -else $as_nop - py_cv_module_resource=missing +else case e in #( + e) py_cv_module_resource=missing ;; +esac fi -else $as_nop - py_cv_module_resource=disabled +else case e in #( + e) py_cv_module_resource=disabled ;; +esac fi fi @@ -30175,11 +31423,13 @@ then : if true then : py_cv_module__scproxy=yes -else $as_nop - py_cv_module__scproxy=missing +else case e in #( + e) py_cv_module__scproxy=missing ;; +esac fi -else $as_nop - py_cv_module__scproxy=disabled +else case e in #( + e) py_cv_module__scproxy=disabled ;; +esac fi fi @@ -30213,11 +31463,13 @@ then : if test "$ac_cv_header_syslog_h" = yes then : py_cv_module_syslog=yes -else $as_nop - py_cv_module_syslog=missing +else case e in #( + e) py_cv_module_syslog=missing ;; +esac fi -else $as_nop - py_cv_module_syslog=disabled +else case e in #( + e) py_cv_module_syslog=disabled ;; +esac fi fi @@ -30251,11 +31503,13 @@ then : if test "$ac_cv_header_termios_h" = yes then : py_cv_module_termios=yes -else $as_nop - py_cv_module_termios=missing +else case e in #( + e) py_cv_module_termios=missing ;; +esac fi -else $as_nop - py_cv_module_termios=disabled +else case e in #( + e) py_cv_module_termios=disabled ;; +esac fi fi @@ -30290,11 +31544,13 @@ then : if test "$ac_cv_header_sys_time_h" = "yes" then : py_cv_module_pyexpat=yes -else $as_nop - py_cv_module_pyexpat=missing +else case e in #( + e) py_cv_module_pyexpat=missing ;; +esac fi -else $as_nop - py_cv_module_pyexpat=disabled +else case e in #( + e) py_cv_module_pyexpat=disabled ;; +esac fi fi @@ -30328,11 +31584,13 @@ then : if true then : py_cv_module__elementtree=yes -else $as_nop - py_cv_module__elementtree=missing +else case e in #( + e) py_cv_module__elementtree=missing ;; +esac fi -else $as_nop - py_cv_module__elementtree=disabled +else case e in #( + e) py_cv_module__elementtree=disabled ;; +esac fi fi @@ -30543,11 +31801,13 @@ then : if true then : py_cv_module__md5=yes -else $as_nop - py_cv_module__md5=missing +else case e in #( + e) py_cv_module__md5=missing ;; +esac fi -else $as_nop - py_cv_module__md5=disabled +else case e in #( + e) py_cv_module__md5=disabled ;; +esac fi fi @@ -30581,11 +31841,13 @@ then : if true then : py_cv_module__sha1=yes -else $as_nop - py_cv_module__sha1=missing +else case e in #( + e) py_cv_module__sha1=missing ;; +esac fi -else $as_nop - py_cv_module__sha1=disabled +else case e in #( + e) py_cv_module__sha1=disabled ;; +esac fi fi @@ -30619,11 +31881,13 @@ then : if true then : py_cv_module__sha2=yes -else $as_nop - py_cv_module__sha2=missing +else case e in #( + e) py_cv_module__sha2=missing ;; +esac fi -else $as_nop - py_cv_module__sha2=disabled +else case e in #( + e) py_cv_module__sha2=disabled ;; +esac fi fi @@ -30657,11 +31921,13 @@ then : if true then : py_cv_module__sha3=yes -else $as_nop - py_cv_module__sha3=missing +else case e in #( + e) py_cv_module__sha3=missing ;; +esac fi -else $as_nop - py_cv_module__sha3=disabled +else case e in #( + e) py_cv_module__sha3=disabled ;; +esac fi fi @@ -30695,11 +31961,13 @@ then : if true then : py_cv_module__blake2=yes -else $as_nop - py_cv_module__blake2=missing +else case e in #( + e) py_cv_module__blake2=missing ;; +esac fi -else $as_nop - py_cv_module__blake2=disabled +else case e in #( + e) py_cv_module__blake2=disabled ;; +esac fi fi @@ -30742,8 +32010,8 @@ printf %s "checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -mss if test ${ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -msse -msse2 -msse3 -msse4.1 -msse4.2" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -30760,11 +32028,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2=yes -else $as_nop - ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2=no +else case e in #( + e) ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2" >&6; } @@ -30793,8 +32063,9 @@ printf "%s\n" "standard" >&6; } fi -else $as_nop - : +else case e in #( + e) : ;; +esac fi fi @@ -30814,8 +32085,8 @@ printf %s "checking whether C compiler accepts -mavx2... " >&6; } if test ${ax_cv_check_cflags__Werror__mavx2+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -Werror -mavx2" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -30832,11 +32103,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags__Werror__mavx2=yes -else $as_nop - ax_cv_check_cflags__Werror__mavx2=no +else case e in #( + e) ax_cv_check_cflags__Werror__mavx2=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__mavx2" >&5 printf "%s\n" "$ax_cv_check_cflags__Werror__mavx2" >&6; } @@ -30864,8 +32137,9 @@ printf "%s\n" "universal2" >&6; } printf "%s\n" "standard" >&6; } fi -else $as_nop - : +else case e in #( + e) : ;; +esac fi fi @@ -30883,11 +32157,13 @@ then : if test "$have_libffi" = yes then : py_cv_module__ctypes=yes -else $as_nop - py_cv_module__ctypes=missing +else case e in #( + e) py_cv_module__ctypes=missing ;; +esac fi -else $as_nop - py_cv_module__ctypes=disabled +else case e in #( + e) py_cv_module__ctypes=disabled ;; +esac fi fi @@ -30921,11 +32197,13 @@ then : if test "$have_curses" = "yes" then : py_cv_module__curses=yes -else $as_nop - py_cv_module__curses=missing +else case e in #( + e) py_cv_module__curses=missing ;; +esac fi -else $as_nop - py_cv_module__curses=disabled +else case e in #( + e) py_cv_module__curses=disabled ;; +esac fi fi @@ -30960,11 +32238,13 @@ then : if test "$have_curses" = "yes" && test "$have_panel" = "yes" then : py_cv_module__curses_panel=yes -else $as_nop - py_cv_module__curses_panel=missing +else case e in #( + e) py_cv_module__curses_panel=missing ;; +esac fi -else $as_nop - py_cv_module__curses_panel=disabled +else case e in #( + e) py_cv_module__curses_panel=disabled ;; +esac fi fi @@ -30999,11 +32279,13 @@ then : if test "$have_mpdec" = "yes" then : py_cv_module__decimal=yes -else $as_nop - py_cv_module__decimal=missing +else case e in #( + e) py_cv_module__decimal=missing ;; +esac fi -else $as_nop - py_cv_module__decimal=disabled +else case e in #( + e) py_cv_module__decimal=disabled ;; +esac fi fi @@ -31037,11 +32319,13 @@ then : if test "$have_dbm" != "no" then : py_cv_module__dbm=yes -else $as_nop - py_cv_module__dbm=missing +else case e in #( + e) py_cv_module__dbm=missing ;; +esac fi -else $as_nop - py_cv_module__dbm=disabled +else case e in #( + e) py_cv_module__dbm=disabled ;; +esac fi fi @@ -31075,11 +32359,13 @@ then : if test "$have_gdbm" = yes then : py_cv_module__gdbm=yes -else $as_nop - py_cv_module__gdbm=missing +else case e in #( + e) py_cv_module__gdbm=missing ;; +esac fi -else $as_nop - py_cv_module__gdbm=disabled +else case e in #( + e) py_cv_module__gdbm=disabled ;; +esac fi fi @@ -31113,11 +32399,13 @@ then : if test "$with_readline" != "no" then : py_cv_module_readline=yes -else $as_nop - py_cv_module_readline=missing +else case e in #( + e) py_cv_module_readline=missing ;; +esac fi -else $as_nop - py_cv_module_readline=disabled +else case e in #( + e) py_cv_module_readline=disabled ;; +esac fi fi @@ -31151,11 +32439,13 @@ then : if test "$have_supported_sqlite3" = "yes" then : py_cv_module__sqlite3=yes -else $as_nop - py_cv_module__sqlite3=missing +else case e in #( + e) py_cv_module__sqlite3=missing ;; +esac fi -else $as_nop - py_cv_module__sqlite3=disabled +else case e in #( + e) py_cv_module__sqlite3=disabled ;; +esac fi fi @@ -31189,11 +32479,13 @@ then : if test "$have_tcltk" = "yes" then : py_cv_module__tkinter=yes -else $as_nop - py_cv_module__tkinter=missing +else case e in #( + e) py_cv_module__tkinter=missing ;; +esac fi -else $as_nop - py_cv_module__tkinter=disabled +else case e in #( + e) py_cv_module__tkinter=disabled ;; +esac fi fi @@ -31227,11 +32519,13 @@ then : if test "$have_uuid" = "yes" then : py_cv_module__uuid=yes -else $as_nop - py_cv_module__uuid=missing +else case e in #( + e) py_cv_module__uuid=missing ;; +esac fi -else $as_nop - py_cv_module__uuid=disabled +else case e in #( + e) py_cv_module__uuid=disabled ;; +esac fi fi @@ -31266,11 +32560,13 @@ then : if test "$have_zlib" = yes then : py_cv_module_zlib=yes -else $as_nop - py_cv_module_zlib=missing +else case e in #( + e) py_cv_module_zlib=missing ;; +esac fi -else $as_nop - py_cv_module_zlib=disabled +else case e in #( + e) py_cv_module_zlib=disabled ;; +esac fi fi @@ -31326,11 +32622,13 @@ then : if test "$have_bzip2" = yes then : py_cv_module__bz2=yes -else $as_nop - py_cv_module__bz2=missing +else case e in #( + e) py_cv_module__bz2=missing ;; +esac fi -else $as_nop - py_cv_module__bz2=disabled +else case e in #( + e) py_cv_module__bz2=disabled ;; +esac fi fi @@ -31364,11 +32662,13 @@ then : if test "$have_liblzma" = yes then : py_cv_module__lzma=yes -else $as_nop - py_cv_module__lzma=missing +else case e in #( + e) py_cv_module__lzma=missing ;; +esac fi -else $as_nop - py_cv_module__lzma=disabled +else case e in #( + e) py_cv_module__lzma=disabled ;; +esac fi fi @@ -31403,11 +32703,13 @@ then : if test "$ac_cv_working_openssl_ssl" = yes then : py_cv_module__ssl=yes -else $as_nop - py_cv_module__ssl=missing +else case e in #( + e) py_cv_module__ssl=missing ;; +esac fi -else $as_nop - py_cv_module__ssl=disabled +else case e in #( + e) py_cv_module__ssl=disabled ;; +esac fi fi @@ -31441,11 +32743,13 @@ then : if test "$ac_cv_working_openssl_hashlib" = yes then : py_cv_module__hashlib=yes -else $as_nop - py_cv_module__hashlib=missing +else case e in #( + e) py_cv_module__hashlib=missing ;; +esac fi -else $as_nop - py_cv_module__hashlib=disabled +else case e in #( + e) py_cv_module__hashlib=disabled ;; +esac fi fi @@ -31480,11 +32784,13 @@ then : if true then : py_cv_module__testcapi=yes -else $as_nop - py_cv_module__testcapi=missing +else case e in #( + e) py_cv_module__testcapi=missing ;; +esac fi -else $as_nop - py_cv_module__testcapi=disabled +else case e in #( + e) py_cv_module__testcapi=disabled ;; +esac fi fi @@ -31518,11 +32824,13 @@ then : if true then : py_cv_module__testclinic=yes -else $as_nop - py_cv_module__testclinic=missing +else case e in #( + e) py_cv_module__testclinic=missing ;; +esac fi -else $as_nop - py_cv_module__testclinic=disabled +else case e in #( + e) py_cv_module__testclinic=disabled ;; +esac fi fi @@ -31556,11 +32864,13 @@ then : if true then : py_cv_module__testclinic_limited=yes -else $as_nop - py_cv_module__testclinic_limited=missing +else case e in #( + e) py_cv_module__testclinic_limited=missing ;; +esac fi -else $as_nop - py_cv_module__testclinic_limited=disabled +else case e in #( + e) py_cv_module__testclinic_limited=disabled ;; +esac fi fi @@ -31594,11 +32904,13 @@ then : if true then : py_cv_module__testlimitedcapi=yes -else $as_nop - py_cv_module__testlimitedcapi=missing +else case e in #( + e) py_cv_module__testlimitedcapi=missing ;; +esac fi -else $as_nop - py_cv_module__testlimitedcapi=disabled +else case e in #( + e) py_cv_module__testlimitedcapi=disabled ;; +esac fi fi @@ -31632,11 +32944,13 @@ then : if true then : py_cv_module__testinternalcapi=yes -else $as_nop - py_cv_module__testinternalcapi=missing +else case e in #( + e) py_cv_module__testinternalcapi=missing ;; +esac fi -else $as_nop - py_cv_module__testinternalcapi=disabled +else case e in #( + e) py_cv_module__testinternalcapi=disabled ;; +esac fi fi @@ -31670,11 +32984,13 @@ then : if true then : py_cv_module__testbuffer=yes -else $as_nop - py_cv_module__testbuffer=missing +else case e in #( + e) py_cv_module__testbuffer=missing ;; +esac fi -else $as_nop - py_cv_module__testbuffer=disabled +else case e in #( + e) py_cv_module__testbuffer=disabled ;; +esac fi fi @@ -31708,11 +33024,13 @@ then : if test "$ac_cv_func_dlopen" = yes then : py_cv_module__testimportmultiple=yes -else $as_nop - py_cv_module__testimportmultiple=missing +else case e in #( + e) py_cv_module__testimportmultiple=missing ;; +esac fi -else $as_nop - py_cv_module__testimportmultiple=disabled +else case e in #( + e) py_cv_module__testimportmultiple=disabled ;; +esac fi fi @@ -31746,11 +33064,13 @@ then : if test "$ac_cv_func_dlopen" = yes then : py_cv_module__testmultiphase=yes -else $as_nop - py_cv_module__testmultiphase=missing +else case e in #( + e) py_cv_module__testmultiphase=missing ;; +esac fi -else $as_nop - py_cv_module__testmultiphase=disabled +else case e in #( + e) py_cv_module__testmultiphase=disabled ;; +esac fi fi @@ -31784,11 +33104,13 @@ then : if test "$ac_cv_func_dlopen" = yes then : py_cv_module__testsinglephase=yes -else $as_nop - py_cv_module__testsinglephase=missing +else case e in #( + e) py_cv_module__testsinglephase=missing ;; +esac fi -else $as_nop - py_cv_module__testsinglephase=disabled +else case e in #( + e) py_cv_module__testsinglephase=disabled ;; +esac fi fi @@ -31822,11 +33144,13 @@ then : if true then : py_cv_module__testexternalinspection=yes -else $as_nop - py_cv_module__testexternalinspection=missing +else case e in #( + e) py_cv_module__testexternalinspection=missing ;; +esac fi -else $as_nop - py_cv_module__testexternalinspection=disabled +else case e in #( + e) py_cv_module__testexternalinspection=disabled ;; +esac fi fi @@ -31860,11 +33184,13 @@ then : if true then : py_cv_module_xxsubtype=yes -else $as_nop - py_cv_module_xxsubtype=missing +else case e in #( + e) py_cv_module_xxsubtype=missing ;; +esac fi -else $as_nop - py_cv_module_xxsubtype=disabled +else case e in #( + e) py_cv_module_xxsubtype=disabled ;; +esac fi fi @@ -31898,11 +33224,13 @@ then : if true then : py_cv_module__xxtestfuzz=yes -else $as_nop - py_cv_module__xxtestfuzz=missing +else case e in #( + e) py_cv_module__xxtestfuzz=missing ;; +esac fi -else $as_nop - py_cv_module__xxtestfuzz=disabled +else case e in #( + e) py_cv_module__xxtestfuzz=disabled ;; +esac fi fi @@ -31936,11 +33264,13 @@ then : if test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes then : py_cv_module__ctypes_test=yes -else $as_nop - py_cv_module__ctypes_test=missing +else case e in #( + e) py_cv_module__ctypes_test=missing ;; +esac fi -else $as_nop - py_cv_module__ctypes_test=disabled +else case e in #( + e) py_cv_module__ctypes_test=disabled ;; +esac fi fi @@ -31975,11 +33305,13 @@ then : if test "$ac_cv_func_dlopen" = yes then : py_cv_module_xxlimited=yes -else $as_nop - py_cv_module_xxlimited=missing +else case e in #( + e) py_cv_module_xxlimited=missing ;; +esac fi -else $as_nop - py_cv_module_xxlimited=disabled +else case e in #( + e) py_cv_module_xxlimited=disabled ;; +esac fi fi @@ -32013,11 +33345,13 @@ then : if test "$ac_cv_func_dlopen" = yes then : py_cv_module_xxlimited_35=yes -else $as_nop - py_cv_module_xxlimited_35=missing +else case e in #( + e) py_cv_module_xxlimited_35=missing ;; +esac fi -else $as_nop - py_cv_module_xxlimited_35=disabled +else case e in #( + e) py_cv_module_xxlimited_35=disabled ;; +esac fi fi @@ -32062,8 +33396,8 @@ cat >confcache <<\_ACEOF # config.status only pays attention to the cache file if you give it # the --recheck option to rerun configure. # -# `ac_cv_env_foo' variables (set or unset) will be overridden when -# loading this file, other *unset* `ac_cv_foo' will be assigned the +# 'ac_cv_env_foo' variables (set or unset) will be overridden when +# loading this file, other *unset* 'ac_cv_foo' will be assigned the # following values. _ACEOF @@ -32093,14 +33427,14 @@ printf "%s\n" "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} (set) 2>&1 | case $as_nl`(ac_space=' '; set) 2>&1` in #( *${as_nl}ac_space=\ *) - # `set' does not quote correctly, so add quotes: double-quote + # 'set' does not quote correctly, so add quotes: double-quote # substitution turns \\\\ into \\, and sed turns \\ into \. sed -n \ "s/'/'\\\\''/g; s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" ;; #( *) - # `set' quotes correctly as required by POSIX, so do not add quotes. + # 'set' quotes correctly as required by POSIX, so do not add quotes. sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" ;; esac | @@ -32519,7 +33853,6 @@ cat >>$CONFIG_STATUS <<\_ASEOF || as_write_fail=1 # Be more Bourne compatible DUALCASE=1; export DUALCASE # for MKS sh -as_nop=: if test ${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1 then : emulate sh @@ -32528,12 +33861,13 @@ then : # is contrary to our usage. Disable this feature. alias -g '${1+"$@"}'='"$@"' setopt NO_GLOB_SUBST -else $as_nop - case `(set -o) 2>/dev/null` in #( +else case e in #( + e) case `(set -o) 2>/dev/null` in #( *posix*) : set -o posix ;; #( *) : ;; +esac ;; esac fi @@ -32605,7 +33939,7 @@ IFS=$as_save_IFS ;; esac -# We did not find ourselves, most probably we were run as `sh COMMAND' +# We did not find ourselves, most probably we were run as 'sh COMMAND' # in which case we are not to be found in the path. if test "x$as_myself" = x; then as_myself=$0 @@ -32634,7 +33968,6 @@ as_fn_error () } # as_fn_error - # as_fn_set_status STATUS # ----------------------- # Set $? to STATUS, without forking. @@ -32674,11 +34007,12 @@ then : { eval $1+=\$2 }' -else $as_nop - as_fn_append () +else case e in #( + e) as_fn_append () { eval $1=\$$1\$2 - } + } ;; +esac fi # as_fn_append # as_fn_arith ARG... @@ -32692,11 +34026,12 @@ then : { as_val=$(( $* )) }' -else $as_nop - as_fn_arith () +else case e in #( + e) as_fn_arith () { as_val=`expr "$@" || test $? -eq 1` - } + } ;; +esac fi # as_fn_arith @@ -32779,9 +34114,9 @@ if (echo >conf$$.file) 2>/dev/null; then if ln -s conf$$.file conf$$ 2>/dev/null; then as_ln_s='ln -s' # ... but there are two gotchas: - # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. - # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. - # In both cases, we have to default to `cp -pR'. + # 1) On MSYS, both 'ln -s file dir' and 'ln file dir' fail. + # 2) DJGPP < 2.04 has no symlinks; 'ln -s' creates a wrapper executable. + # In both cases, we have to default to 'cp -pR'. ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || as_ln_s='cp -pR' elif ln conf$$.file conf$$ 2>/dev/null; then @@ -32862,10 +34197,12 @@ as_test_x='test -x' as_executable_p=as_fn_executable_p # Sed expression to map a string onto a valid CPP name. -as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" +as_sed_cpp="y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g" +as_tr_cpp="eval sed '$as_sed_cpp'" # deprecated # Sed expression to map a string onto a valid variable name. -as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" +as_sed_sh="y%*+%pp%;s%[^_$as_cr_alnum]%_%g" +as_tr_sh="eval sed '$as_sed_sh'" # deprecated exec 6>&1 @@ -32881,7 +34218,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # values after options handling. ac_log=" This file was extended by python $as_me 3.14, which was -generated by GNU Autoconf 2.71. Invocation command line was +generated by GNU Autoconf 2.72. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS @@ -32912,7 +34249,7 @@ _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 ac_cs_usage="\ -\`$as_me' instantiates files and other configuration actions +'$as_me' instantiates files and other configuration actions from templates according to the current configuration. Unless the files and actions are specified as TAGs, all are instantiated by default. @@ -32945,10 +34282,10 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ python config.status 3.14 -configured by $0, generated by GNU Autoconf 2.71, +configured by $0, generated by GNU Autoconf 2.72, with options \\"\$ac_cs_config\\" -Copyright (C) 2021 Free Software Foundation, Inc. +Copyright (C) 2023 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it." @@ -33009,8 +34346,8 @@ do ac_need_defaults=false;; --he | --h) # Conflict between --help and --header - as_fn_error $? "ambiguous option: \`$1' -Try \`$0 --help' for more information.";; + as_fn_error $? "ambiguous option: '$1' +Try '$0 --help' for more information.";; --help | --hel | -h ) printf "%s\n" "$ac_cs_usage"; exit ;; -q | -quiet | --quiet | --quie | --qui | --qu | --q \ @@ -33018,8 +34355,8 @@ Try \`$0 --help' for more information.";; ac_cs_silent=: ;; # This is an error. - -*) as_fn_error $? "unrecognized option: \`$1' -Try \`$0 --help' for more information." ;; + -*) as_fn_error $? "unrecognized option: '$1' +Try '$0 --help' for more information." ;; *) as_fn_append ac_config_targets " $1" ac_need_defaults=false ;; @@ -33081,7 +34418,7 @@ do "Modules/Setup.stdlib") CONFIG_FILES="$CONFIG_FILES Modules/Setup.stdlib" ;; "Modules/ld_so_aix") CONFIG_FILES="$CONFIG_FILES Modules/ld_so_aix" ;; - *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; + *) as_fn_error $? "invalid argument: '$ac_config_target'" "$LINENO" 5;; esac done @@ -33100,7 +34437,7 @@ fi # creating and moving files from /tmp can sometimes cause problems. # Hook for its removal unless debugging. # Note that there is a small window in which the directory will not be cleaned: -# after its creation but before its name has been assigned to `$tmp'. +# after its creation but before its name has been assigned to '$tmp'. $debug || { tmp= ac_tmp= @@ -33124,7 +34461,7 @@ ac_tmp=$tmp # Set up the scripts for CONFIG_FILES section. # No need to generate them if there are no CONFIG_FILES. -# This happens for instance with `./config.status config.h'. +# This happens for instance with './config.status config.h'. if test -n "$CONFIG_FILES"; then @@ -33282,13 +34619,13 @@ fi # test -n "$CONFIG_FILES" # Set up the scripts for CONFIG_HEADERS section. # No need to generate them if there are no CONFIG_HEADERS. -# This happens for instance with `./config.status Makefile'. +# This happens for instance with './config.status Makefile'. if test -n "$CONFIG_HEADERS"; then cat >"$ac_tmp/defines.awk" <<\_ACAWK || BEGIN { _ACEOF -# Transform confdefs.h into an awk script `defines.awk', embedded as +# Transform confdefs.h into an awk script 'defines.awk', embedded as # here-document in config.status, that substitutes the proper values into # config.h.in to produce config.h. @@ -33398,7 +34735,7 @@ do esac case $ac_mode$ac_tag in :[FHL]*:*);; - :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;; + :L* | :C*:*) as_fn_error $? "invalid tag '$ac_tag'" "$LINENO" 5;; :[FH]-) ac_tag=-:-;; :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; esac @@ -33420,19 +34757,19 @@ do -) ac_f="$ac_tmp/stdin";; *) # Look for the file first in the build tree, then in the source tree # (if the path is not absolute). The absolute path cannot be DOS-style, - # because $ac_f cannot contain `:'. + # because $ac_f cannot contain ':'. test -f "$ac_f" || case $ac_f in [\\/$]*) false;; *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; esac || - as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;; + as_fn_error 1 "cannot find input file: '$ac_f'" "$LINENO" 5;; esac case $ac_f in *\'*) ac_f=`printf "%s\n" "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac as_fn_append ac_file_inputs " '$ac_f'" done - # Let's still pretend it is `configure' which instantiates (i.e., don't + # Let's still pretend it is 'configure' which instantiates (i.e., don't # use $as_me), people would be surprised to read: # /* config.h. Generated by config.status. */ configure_input='Generated from '` @@ -33565,7 +34902,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 esac _ACEOF -# Neutralize VPATH when `$srcdir' = `.'. +# Neutralize VPATH when '$srcdir' = '.'. # Shell code in configure.ac might set extrasub. # FIXME: do we really want to maintain this feature? cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 @@ -33596,9 +34933,9 @@ test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } && { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' \ "$ac_tmp/out"`; test -z "$ac_out"; } && - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir' + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable 'datarootdir' which seems to be undefined. Please make sure it is defined" >&5 -printf "%s\n" "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' +printf "%s\n" "$as_me: WARNING: $ac_file contains a reference to the variable 'datarootdir' which seems to be undefined. Please make sure it is defined" >&2;} rm -f "$ac_tmp/stdin" diff --git a/install-sh b/install-sh index ec298b53740270..7c56c9c0151036 100755 --- a/install-sh +++ b/install-sh @@ -1,7 +1,7 @@ #!/bin/sh # install - install a program, script, or datafile -scriptversion=2020-11-14.01; # UTC +scriptversion=2023-11-23.18; # UTC # This originates from X11R5 (mit/util/scripts/install.sh), which was # later released in X11R6 (xc/config/util/install.sh) with the @@ -124,9 +124,9 @@ it's up to you to specify -f if you want it. If -S is not specified, no backups are attempted. -Email bug reports to bug-automake@gnu.org. -Automake home page: https://www.gnu.org/software/automake/ -" +Report bugs to . +GNU Automake home page: . +General help using GNU software: ." while test $# -ne 0; do case $1 in diff --git a/pyconfig.h.in b/pyconfig.h.in index ca08f087a85e9f..d862966b7de4c8 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -16,13 +16,13 @@ support for AIX C++ shared extension modules. */ #undef AIX_GENUINE_CPLUSPLUS -/* The normal alignment of `long', in bytes. */ +/* The normal alignment of 'long', in bytes. */ #undef ALIGNOF_LONG -/* The normal alignment of `max_align_t', in bytes. */ +/* The normal alignment of 'max_align_t', in bytes. */ #undef ALIGNOF_MAX_ALIGN_T -/* The normal alignment of `size_t', in bytes. */ +/* The normal alignment of 'size_t', in bytes. */ #undef ALIGNOF_SIZE_T /* Alternative SOABI used in debug build to load C extensions built in release @@ -59,16 +59,16 @@ /* Define if you have the 'accept' function. */ #undef HAVE_ACCEPT -/* Define to 1 if you have the `accept4' function. */ +/* Define to 1 if you have the 'accept4' function. */ #undef HAVE_ACCEPT4 -/* Define to 1 if you have the `acosh' function. */ +/* Define to 1 if you have the 'acosh' function. */ #undef HAVE_ACOSH /* struct addrinfo (netdb.h) */ #undef HAVE_ADDRINFO -/* Define to 1 if you have the `alarm' function. */ +/* Define to 1 if you have the 'alarm' function. */ #undef HAVE_ALARM /* Define if aligned memory access is required */ @@ -80,19 +80,19 @@ /* Define this if your time.h defines altzone. */ #undef HAVE_ALTZONE -/* Define to 1 if you have the `asinh' function. */ +/* Define to 1 if you have the 'asinh' function. */ #undef HAVE_ASINH /* Define to 1 if you have the header file. */ #undef HAVE_ASM_TYPES_H -/* Define to 1 if you have the `atanh' function. */ +/* Define to 1 if you have the 'atanh' function. */ #undef HAVE_ATANH /* Define if you have the 'bind' function. */ #undef HAVE_BIND -/* Define to 1 if you have the `bind_textdomain_codeset' function. */ +/* Define to 1 if you have the 'bind_textdomain_codeset' function. */ #undef HAVE_BIND_TEXTDOMAIN_CODESET /* Define to 1 if you have the header file. */ @@ -135,43 +135,43 @@ /* Define to 1 if you have the 'chflags' function. */ #undef HAVE_CHFLAGS -/* Define to 1 if you have the `chmod' function. */ +/* Define to 1 if you have the 'chmod' function. */ #undef HAVE_CHMOD -/* Define to 1 if you have the `chown' function. */ +/* Define to 1 if you have the 'chown' function. */ #undef HAVE_CHOWN /* Define if you have the 'chroot' function. */ #undef HAVE_CHROOT -/* Define to 1 if you have the `clock' function. */ +/* Define to 1 if you have the 'clock' function. */ #undef HAVE_CLOCK -/* Define to 1 if you have the `clock_getres' function. */ +/* Define to 1 if you have the 'clock_getres' function. */ #undef HAVE_CLOCK_GETRES -/* Define to 1 if you have the `clock_gettime' function. */ +/* Define to 1 if you have the 'clock_gettime' function. */ #undef HAVE_CLOCK_GETTIME -/* Define to 1 if you have the `clock_nanosleep' function. */ +/* Define to 1 if you have the 'clock_nanosleep' function. */ #undef HAVE_CLOCK_NANOSLEEP -/* Define to 1 if you have the `clock_settime' function. */ +/* Define to 1 if you have the 'clock_settime' function. */ #undef HAVE_CLOCK_SETTIME -/* Define to 1 if the system has the type `clock_t'. */ +/* Define to 1 if the system has the type 'clock_t'. */ #undef HAVE_CLOCK_T -/* Define to 1 if you have the `closefrom' function. */ +/* Define to 1 if you have the 'closefrom' function. */ #undef HAVE_CLOSEFROM -/* Define to 1 if you have the `close_range' function. */ +/* Define to 1 if you have the 'close_range' function. */ #undef HAVE_CLOSE_RANGE /* Define if the C compiler supports computed gotos. */ #undef HAVE_COMPUTED_GOTOS -/* Define to 1 if you have the `confstr' function. */ +/* Define to 1 if you have the 'confstr' function. */ #undef HAVE_CONFSTR /* Define to 1 if you have the header file. */ @@ -180,10 +180,10 @@ /* Define if you have the 'connect' function. */ #undef HAVE_CONNECT -/* Define to 1 if you have the `copy_file_range' function. */ +/* Define to 1 if you have the 'copy_file_range' function. */ #undef HAVE_COPY_FILE_RANGE -/* Define to 1 if you have the `ctermid' function. */ +/* Define to 1 if you have the 'ctermid' function. */ #undef HAVE_CTERMID /* Define if you have the 'ctermid_r' function. */ @@ -228,39 +228,39 @@ /* Define to 1 if you have the header file. */ #undef HAVE_DB_H -/* Define to 1 if you have the declaration of `RTLD_DEEPBIND', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_DEEPBIND', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_DEEPBIND -/* Define to 1 if you have the declaration of `RTLD_GLOBAL', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_GLOBAL', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_GLOBAL -/* Define to 1 if you have the declaration of `RTLD_LAZY', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_LAZY', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_LAZY -/* Define to 1 if you have the declaration of `RTLD_LOCAL', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_LOCAL', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_LOCAL -/* Define to 1 if you have the declaration of `RTLD_MEMBER', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_MEMBER', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_MEMBER -/* Define to 1 if you have the declaration of `RTLD_NODELETE', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_NODELETE', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_NODELETE -/* Define to 1 if you have the declaration of `RTLD_NOLOAD', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_NOLOAD', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_NOLOAD -/* Define to 1 if you have the declaration of `RTLD_NOW', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_NOW', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_NOW -/* Define to 1 if you have the declaration of `tzname', and to 0 if you don't. +/* Define to 1 if you have the declaration of 'tzname', and to 0 if you don't. */ #undef HAVE_DECL_TZNAME @@ -279,7 +279,7 @@ /* Define to 1 if the dirent structure has a d_type field */ #undef HAVE_DIRENT_D_TYPE -/* Define to 1 if you have the header file, and it defines `DIR'. +/* Define to 1 if you have the header file, and it defines 'DIR'. */ #undef HAVE_DIRENT_H @@ -289,16 +289,16 @@ /* Define to 1 if you have the header file. */ #undef HAVE_DLFCN_H -/* Define to 1 if you have the `dlopen' function. */ +/* Define to 1 if you have the 'dlopen' function. */ #undef HAVE_DLOPEN -/* Define to 1 if you have the `dup' function. */ +/* Define to 1 if you have the 'dup' function. */ #undef HAVE_DUP -/* Define to 1 if you have the `dup2' function. */ +/* Define to 1 if you have the 'dup2' function. */ #undef HAVE_DUP2 -/* Define to 1 if you have the `dup3' function. */ +/* Define to 1 if you have the 'dup3' function. */ #undef HAVE_DUP3 /* Define if you have the '_dyld_shared_cache_contains_path' function. */ @@ -319,10 +319,10 @@ /* Define if you have the 'epoll_create1' function. */ #undef HAVE_EPOLL_CREATE1 -/* Define to 1 if you have the `erf' function. */ +/* Define to 1 if you have the 'erf' function. */ #undef HAVE_ERF -/* Define to 1 if you have the `erfc' function. */ +/* Define to 1 if you have the 'erfc' function. */ #undef HAVE_ERFC /* Define to 1 if you have the header file. */ @@ -331,34 +331,34 @@ /* Define if you have the 'eventfd' function. */ #undef HAVE_EVENTFD -/* Define to 1 if you have the `execv' function. */ +/* Define to 1 if you have the 'execv' function. */ #undef HAVE_EXECV -/* Define to 1 if you have the `explicit_bzero' function. */ +/* Define to 1 if you have the 'explicit_bzero' function. */ #undef HAVE_EXPLICIT_BZERO -/* Define to 1 if you have the `explicit_memset' function. */ +/* Define to 1 if you have the 'explicit_memset' function. */ #undef HAVE_EXPLICIT_MEMSET -/* Define to 1 if you have the `expm1' function. */ +/* Define to 1 if you have the 'expm1' function. */ #undef HAVE_EXPM1 -/* Define to 1 if you have the `faccessat' function. */ +/* Define to 1 if you have the 'faccessat' function. */ #undef HAVE_FACCESSAT /* Define if you have the 'fchdir' function. */ #undef HAVE_FCHDIR -/* Define to 1 if you have the `fchmod' function. */ +/* Define to 1 if you have the 'fchmod' function. */ #undef HAVE_FCHMOD -/* Define to 1 if you have the `fchmodat' function. */ +/* Define to 1 if you have the 'fchmodat' function. */ #undef HAVE_FCHMODAT -/* Define to 1 if you have the `fchown' function. */ +/* Define to 1 if you have the 'fchown' function. */ #undef HAVE_FCHOWN -/* Define to 1 if you have the `fchownat' function. */ +/* Define to 1 if you have the 'fchownat' function. */ #undef HAVE_FCHOWNAT /* Define to 1 if you have the header file. */ @@ -367,13 +367,13 @@ /* Define if you have the 'fdatasync' function. */ #undef HAVE_FDATASYNC -/* Define to 1 if you have the `fdopendir' function. */ +/* Define to 1 if you have the 'fdopendir' function. */ #undef HAVE_FDOPENDIR -/* Define to 1 if you have the `fdwalk' function. */ +/* Define to 1 if you have the 'fdwalk' function. */ #undef HAVE_FDWALK -/* Define to 1 if you have the `fexecve' function. */ +/* Define to 1 if you have the 'fexecve' function. */ #undef HAVE_FEXECVE /* Define if you have the 'ffi_closure_alloc' function. */ @@ -385,58 +385,58 @@ /* Define if you have the 'ffi_prep_closure_loc' function. */ #undef HAVE_FFI_PREP_CLOSURE_LOC -/* Define to 1 if you have the `flock' function. */ +/* Define to 1 if you have the 'flock' function. */ #undef HAVE_FLOCK -/* Define to 1 if you have the `fork' function. */ +/* Define to 1 if you have the 'fork' function. */ #undef HAVE_FORK -/* Define to 1 if you have the `fork1' function. */ +/* Define to 1 if you have the 'fork1' function. */ #undef HAVE_FORK1 -/* Define to 1 if you have the `forkpty' function. */ +/* Define to 1 if you have the 'forkpty' function. */ #undef HAVE_FORKPTY -/* Define to 1 if you have the `fpathconf' function. */ +/* Define to 1 if you have the 'fpathconf' function. */ #undef HAVE_FPATHCONF -/* Define to 1 if you have the `fseek64' function. */ +/* Define to 1 if you have the 'fseek64' function. */ #undef HAVE_FSEEK64 -/* Define to 1 if you have the `fseeko' function. */ +/* Define to 1 if you have the 'fseeko' function. */ #undef HAVE_FSEEKO -/* Define to 1 if you have the `fstatat' function. */ +/* Define to 1 if you have the 'fstatat' function. */ #undef HAVE_FSTATAT -/* Define to 1 if you have the `fstatvfs' function. */ +/* Define to 1 if you have the 'fstatvfs' function. */ #undef HAVE_FSTATVFS /* Define if you have the 'fsync' function. */ #undef HAVE_FSYNC -/* Define to 1 if you have the `ftell64' function. */ +/* Define to 1 if you have the 'ftell64' function. */ #undef HAVE_FTELL64 -/* Define to 1 if you have the `ftello' function. */ +/* Define to 1 if you have the 'ftello' function. */ #undef HAVE_FTELLO -/* Define to 1 if you have the `ftime' function. */ +/* Define to 1 if you have the 'ftime' function. */ #undef HAVE_FTIME -/* Define to 1 if you have the `ftruncate' function. */ +/* Define to 1 if you have the 'ftruncate' function. */ #undef HAVE_FTRUNCATE -/* Define to 1 if you have the `futimens' function. */ +/* Define to 1 if you have the 'futimens' function. */ #undef HAVE_FUTIMENS -/* Define to 1 if you have the `futimes' function. */ +/* Define to 1 if you have the 'futimes' function. */ #undef HAVE_FUTIMES -/* Define to 1 if you have the `futimesat' function. */ +/* Define to 1 if you have the 'futimesat' function. */ #undef HAVE_FUTIMESAT -/* Define to 1 if you have the `gai_strerror' function. */ +/* Define to 1 if you have the 'gai_strerror' function. */ #undef HAVE_GAI_STRERROR /* Define if we can use gcc inline assembler to get and set mc68881 fpcr */ @@ -467,40 +467,40 @@ /* Define this if you have flockfile(), getc_unlocked(), and funlockfile() */ #undef HAVE_GETC_UNLOCKED -/* Define to 1 if you have the `getegid' function. */ +/* Define to 1 if you have the 'getegid' function. */ #undef HAVE_GETEGID -/* Define to 1 if you have the `getentropy' function. */ +/* Define to 1 if you have the 'getentropy' function. */ #undef HAVE_GETENTROPY -/* Define to 1 if you have the `geteuid' function. */ +/* Define to 1 if you have the 'geteuid' function. */ #undef HAVE_GETEUID -/* Define to 1 if you have the `getgid' function. */ +/* Define to 1 if you have the 'getgid' function. */ #undef HAVE_GETGID -/* Define to 1 if you have the `getgrent' function. */ +/* Define to 1 if you have the 'getgrent' function. */ #undef HAVE_GETGRENT -/* Define to 1 if you have the `getgrgid' function. */ +/* Define to 1 if you have the 'getgrgid' function. */ #undef HAVE_GETGRGID -/* Define to 1 if you have the `getgrgid_r' function. */ +/* Define to 1 if you have the 'getgrgid_r' function. */ #undef HAVE_GETGRGID_R -/* Define to 1 if you have the `getgrnam_r' function. */ +/* Define to 1 if you have the 'getgrnam_r' function. */ #undef HAVE_GETGRNAM_R -/* Define to 1 if you have the `getgrouplist' function. */ +/* Define to 1 if you have the 'getgrouplist' function. */ #undef HAVE_GETGROUPLIST -/* Define to 1 if you have the `getgroups' function. */ +/* Define to 1 if you have the 'getgroups' function. */ #undef HAVE_GETGROUPS /* Define if you have the 'gethostbyaddr' function. */ #undef HAVE_GETHOSTBYADDR -/* Define to 1 if you have the `gethostbyname' function. */ +/* Define to 1 if you have the 'gethostbyname' function. */ #undef HAVE_GETHOSTBYNAME /* Define this if you have some version of gethostbyname_r() */ @@ -515,19 +515,19 @@ /* Define this if you have the 6-arg version of gethostbyname_r(). */ #undef HAVE_GETHOSTBYNAME_R_6_ARG -/* Define to 1 if you have the `gethostname' function. */ +/* Define to 1 if you have the 'gethostname' function. */ #undef HAVE_GETHOSTNAME -/* Define to 1 if you have the `getitimer' function. */ +/* Define to 1 if you have the 'getitimer' function. */ #undef HAVE_GETITIMER -/* Define to 1 if you have the `getloadavg' function. */ +/* Define to 1 if you have the 'getloadavg' function. */ #undef HAVE_GETLOADAVG -/* Define to 1 if you have the `getlogin' function. */ +/* Define to 1 if you have the 'getlogin' function. */ #undef HAVE_GETLOGIN -/* Define to 1 if you have the `getnameinfo' function. */ +/* Define to 1 if you have the 'getnameinfo' function. */ #undef HAVE_GETNAMEINFO /* Define if you have the 'getpagesize' function. */ @@ -536,34 +536,34 @@ /* Define if you have the 'getpeername' function. */ #undef HAVE_GETPEERNAME -/* Define to 1 if you have the `getpgid' function. */ +/* Define to 1 if you have the 'getpgid' function. */ #undef HAVE_GETPGID -/* Define to 1 if you have the `getpgrp' function. */ +/* Define to 1 if you have the 'getpgrp' function. */ #undef HAVE_GETPGRP -/* Define to 1 if you have the `getpid' function. */ +/* Define to 1 if you have the 'getpid' function. */ #undef HAVE_GETPID -/* Define to 1 if you have the `getppid' function. */ +/* Define to 1 if you have the 'getppid' function. */ #undef HAVE_GETPPID -/* Define to 1 if you have the `getpriority' function. */ +/* Define to 1 if you have the 'getpriority' function. */ #undef HAVE_GETPRIORITY /* Define if you have the 'getprotobyname' function. */ #undef HAVE_GETPROTOBYNAME -/* Define to 1 if you have the `getpwent' function. */ +/* Define to 1 if you have the 'getpwent' function. */ #undef HAVE_GETPWENT -/* Define to 1 if you have the `getpwnam_r' function. */ +/* Define to 1 if you have the 'getpwnam_r' function. */ #undef HAVE_GETPWNAM_R -/* Define to 1 if you have the `getpwuid' function. */ +/* Define to 1 if you have the 'getpwuid' function. */ #undef HAVE_GETPWUID -/* Define to 1 if you have the `getpwuid_r' function. */ +/* Define to 1 if you have the 'getpwuid_r' function. */ #undef HAVE_GETPWUID_R /* Define to 1 if the getrandom() function is available */ @@ -572,13 +572,13 @@ /* Define to 1 if the Linux getrandom() syscall is available */ #undef HAVE_GETRANDOM_SYSCALL -/* Define to 1 if you have the `getresgid' function. */ +/* Define to 1 if you have the 'getresgid' function. */ #undef HAVE_GETRESGID -/* Define to 1 if you have the `getresuid' function. */ +/* Define to 1 if you have the 'getresuid' function. */ #undef HAVE_GETRESUID -/* Define to 1 if you have the `getrusage' function. */ +/* Define to 1 if you have the 'getrusage' function. */ #undef HAVE_GETRUSAGE /* Define if you have the 'getservbyname' function. */ @@ -587,29 +587,29 @@ /* Define if you have the 'getservbyport' function. */ #undef HAVE_GETSERVBYPORT -/* Define to 1 if you have the `getsid' function. */ +/* Define to 1 if you have the 'getsid' function. */ #undef HAVE_GETSID /* Define if you have the 'getsockname' function. */ #undef HAVE_GETSOCKNAME -/* Define to 1 if you have the `getspent' function. */ +/* Define to 1 if you have the 'getspent' function. */ #undef HAVE_GETSPENT -/* Define to 1 if you have the `getspnam' function. */ +/* Define to 1 if you have the 'getspnam' function. */ #undef HAVE_GETSPNAM -/* Define to 1 if you have the `getuid' function. */ +/* Define to 1 if you have the 'getuid' function. */ #undef HAVE_GETUID -/* Define to 1 if you have the `getwd' function. */ +/* Define to 1 if you have the 'getwd' function. */ #undef HAVE_GETWD /* Define if glibc has incorrect _FORTIFY_SOURCE wrappers for memmove and bcopy. */ #undef HAVE_GLIBC_MEMMOVE_BUG -/* Define to 1 if you have the `grantpt' function. */ +/* Define to 1 if you have the 'grantpt' function. */ #undef HAVE_GRANTPT /* Define to 1 if you have the header file. */ @@ -621,7 +621,7 @@ /* Define this if you have le64toh() */ #undef HAVE_HTOLE64 -/* Define to 1 if you have the `if_nameindex' function. */ +/* Define to 1 if you have the 'if_nameindex' function. */ #undef HAVE_IF_NAMEINDEX /* Define if you have the 'inet_aton' function. */ @@ -633,7 +633,7 @@ /* Define if you have the 'inet_pton' function. */ #undef HAVE_INET_PTON -/* Define to 1 if you have the `initgroups' function. */ +/* Define to 1 if you have the 'initgroups' function. */ #undef HAVE_INITGROUPS /* Define to 1 if you have the header file. */ @@ -645,10 +645,10 @@ /* Define if gcc has the ipa-pure-const bug. */ #undef HAVE_IPA_PURE_CONST_BUG -/* Define to 1 if you have the `kill' function. */ +/* Define to 1 if you have the 'kill' function. */ #undef HAVE_KILL -/* Define to 1 if you have the `killpg' function. */ +/* Define to 1 if you have the 'killpg' function. */ #undef HAVE_KILLPG /* Define if you have the 'kqueue' function. */ @@ -666,31 +666,31 @@ /* Define to 1 if you have the 'lchflags' function. */ #undef HAVE_LCHFLAGS -/* Define to 1 if you have the `lchmod' function. */ +/* Define to 1 if you have the 'lchmod' function. */ #undef HAVE_LCHMOD -/* Define to 1 if you have the `lchown' function. */ +/* Define to 1 if you have the 'lchown' function. */ #undef HAVE_LCHOWN /* Define to 1 if you have the `db' library (-ldb). */ #undef HAVE_LIBDB -/* Define to 1 if you have the `dl' library (-ldl). */ +/* Define to 1 if you have the 'dl' library (-ldl). */ #undef HAVE_LIBDL -/* Define to 1 if you have the `dld' library (-ldld). */ +/* Define to 1 if you have the 'dld' library (-ldld). */ #undef HAVE_LIBDLD -/* Define to 1 if you have the `ieee' library (-lieee). */ +/* Define to 1 if you have the 'ieee' library (-lieee). */ #undef HAVE_LIBIEEE /* Define to 1 if you have the header file. */ #undef HAVE_LIBINTL_H -/* Define to 1 if you have the `sendfile' library (-lsendfile). */ +/* Define to 1 if you have the 'sendfile' library (-lsendfile). */ #undef HAVE_LIBSENDFILE -/* Define to 1 if you have the `sqlite3' library (-lsqlite3). */ +/* Define to 1 if you have the 'sqlite3' library (-lsqlite3). */ #undef HAVE_LIBSQLITE3 /* Define to 1 if you have the header file. */ @@ -699,7 +699,7 @@ /* Define if you have the 'link' function. */ #undef HAVE_LINK -/* Define to 1 if you have the `linkat' function. */ +/* Define to 1 if you have the 'linkat' function. */ #undef HAVE_LINKAT /* Define to 1 if you have the header file. */ @@ -762,73 +762,73 @@ /* Define if you have the 'listen' function. */ #undef HAVE_LISTEN -/* Define to 1 if you have the `lockf' function. */ +/* Define to 1 if you have the 'lockf' function. */ #undef HAVE_LOCKF -/* Define to 1 if you have the `log1p' function. */ +/* Define to 1 if you have the 'log1p' function. */ #undef HAVE_LOG1P -/* Define to 1 if you have the `log2' function. */ +/* Define to 1 if you have the 'log2' function. */ #undef HAVE_LOG2 /* Define to 1 if you have the `login_tty' function. */ #undef HAVE_LOGIN_TTY -/* Define to 1 if the system has the type `long double'. */ +/* Define to 1 if the system has the type 'long double'. */ #undef HAVE_LONG_DOUBLE -/* Define to 1 if you have the `lstat' function. */ +/* Define to 1 if you have the 'lstat' function. */ #undef HAVE_LSTAT -/* Define to 1 if you have the `lutimes' function. */ +/* Define to 1 if you have the 'lutimes' function. */ #undef HAVE_LUTIMES /* Define to 1 if you have the header file. */ #undef HAVE_LZMA_H -/* Define to 1 if you have the `madvise' function. */ +/* Define to 1 if you have the 'madvise' function. */ #undef HAVE_MADVISE /* Define this if you have the makedev macro. */ #undef HAVE_MAKEDEV -/* Define to 1 if you have the `mbrtowc' function. */ +/* Define to 1 if you have the 'mbrtowc' function. */ #undef HAVE_MBRTOWC /* Define if you have the 'memfd_create' function. */ #undef HAVE_MEMFD_CREATE -/* Define to 1 if you have the `memrchr' function. */ +/* Define to 1 if you have the 'memrchr' function. */ #undef HAVE_MEMRCHR /* Define to 1 if you have the header file. */ #undef HAVE_MINIX_CONFIG_H -/* Define to 1 if you have the `mkdirat' function. */ +/* Define to 1 if you have the 'mkdirat' function. */ #undef HAVE_MKDIRAT -/* Define to 1 if you have the `mkfifo' function. */ +/* Define to 1 if you have the 'mkfifo' function. */ #undef HAVE_MKFIFO -/* Define to 1 if you have the `mkfifoat' function. */ +/* Define to 1 if you have the 'mkfifoat' function. */ #undef HAVE_MKFIFOAT -/* Define to 1 if you have the `mknod' function. */ +/* Define to 1 if you have the 'mknod' function. */ #undef HAVE_MKNOD -/* Define to 1 if you have the `mknodat' function. */ +/* Define to 1 if you have the 'mknodat' function. */ #undef HAVE_MKNODAT -/* Define to 1 if you have the `mktime' function. */ +/* Define to 1 if you have the 'mktime' function. */ #undef HAVE_MKTIME -/* Define to 1 if you have the `mmap' function. */ +/* Define to 1 if you have the 'mmap' function. */ #undef HAVE_MMAP -/* Define to 1 if you have the `mremap' function. */ +/* Define to 1 if you have the 'mremap' function. */ #undef HAVE_MREMAP -/* Define to 1 if you have the `nanosleep' function. */ +/* Define to 1 if you have the 'nanosleep' function. */ #undef HAVE_NANOSLEEP /* Define if you have the 'ncurses' library */ @@ -861,7 +861,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_NDBM_H -/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* Define to 1 if you have the header file, and it defines 'DIR'. */ #undef HAVE_NDIR_H /* Define to 1 if you have the header file. */ @@ -885,20 +885,20 @@ /* Define to 1 if you have the header file. */ #undef HAVE_NET_IF_H -/* Define to 1 if you have the `nice' function. */ +/* Define to 1 if you have the 'nice' function. */ #undef HAVE_NICE /* Define if the internal form of wchar_t in non-Unicode locales is not Unicode. */ #undef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION -/* Define to 1 if you have the `openat' function. */ +/* Define to 1 if you have the 'openat' function. */ #undef HAVE_OPENAT -/* Define to 1 if you have the `opendir' function. */ +/* Define to 1 if you have the 'opendir' function. */ #undef HAVE_OPENDIR -/* Define to 1 if you have the `openpty' function. */ +/* Define to 1 if you have the 'openpty' function. */ #undef HAVE_OPENPTY /* Define if you have the 'panel' library */ @@ -910,53 +910,53 @@ /* Define to 1 if you have the header file. */ #undef HAVE_PANEL_H -/* Define to 1 if you have the `pathconf' function. */ +/* Define to 1 if you have the 'pathconf' function. */ #undef HAVE_PATHCONF -/* Define to 1 if you have the `pause' function. */ +/* Define to 1 if you have the 'pause' function. */ #undef HAVE_PAUSE -/* Define to 1 if you have the `pipe' function. */ +/* Define to 1 if you have the 'pipe' function. */ #undef HAVE_PIPE -/* Define to 1 if you have the `pipe2' function. */ +/* Define to 1 if you have the 'pipe2' function. */ #undef HAVE_PIPE2 -/* Define to 1 if you have the `plock' function. */ +/* Define to 1 if you have the 'plock' function. */ #undef HAVE_PLOCK -/* Define to 1 if you have the `poll' function. */ +/* Define to 1 if you have the 'poll' function. */ #undef HAVE_POLL /* Define to 1 if you have the header file. */ #undef HAVE_POLL_H -/* Define to 1 if you have the `posix_fadvise' function. */ +/* Define to 1 if you have the 'posix_fadvise' function. */ #undef HAVE_POSIX_FADVISE -/* Define to 1 if you have the `posix_fallocate' function. */ +/* Define to 1 if you have the 'posix_fallocate' function. */ #undef HAVE_POSIX_FALLOCATE -/* Define to 1 if you have the `posix_openpt' function. */ +/* Define to 1 if you have the 'posix_openpt' function. */ #undef HAVE_POSIX_OPENPT -/* Define to 1 if you have the `posix_spawn' function. */ +/* Define to 1 if you have the 'posix_spawn' function. */ #undef HAVE_POSIX_SPAWN -/* Define to 1 if you have the `posix_spawnp' function. */ +/* Define to 1 if you have the 'posix_spawnp' function. */ #undef HAVE_POSIX_SPAWNP -/* Define to 1 if you have the `posix_spawn_file_actions_addclosefrom_np' +/* Define to 1 if you have the 'posix_spawn_file_actions_addclosefrom_np' function. */ #undef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP -/* Define to 1 if you have the `pread' function. */ +/* Define to 1 if you have the 'pread' function. */ #undef HAVE_PREAD -/* Define to 1 if you have the `preadv' function. */ +/* Define to 1 if you have the 'preadv' function. */ #undef HAVE_PREADV -/* Define to 1 if you have the `preadv2' function. */ +/* Define to 1 if you have the 'preadv2' function. */ #undef HAVE_PREADV2 /* Define if you have the 'prlimit' function. */ @@ -965,83 +965,83 @@ /* Define to 1 if you have the header file. */ #undef HAVE_PROCESS_H -/* Define to 1 if you have the `process_vm_readv' function. */ +/* Define to 1 if you have the 'process_vm_readv' function. */ #undef HAVE_PROCESS_VM_READV /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES -/* Define to 1 if you have the `pthread_condattr_setclock' function. */ +/* Define to 1 if you have the 'pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK -/* Define to 1 if you have the `pthread_cond_timedwait_relative_np' function. +/* Define to 1 if you have the 'pthread_cond_timedwait_relative_np' function. */ #undef HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP /* Defined for Solaris 2.6 bug in pthread header. */ #undef HAVE_PTHREAD_DESTRUCTOR -/* Define to 1 if you have the `pthread_getcpuclockid' function. */ +/* Define to 1 if you have the 'pthread_getcpuclockid' function. */ #undef HAVE_PTHREAD_GETCPUCLOCKID -/* Define to 1 if you have the `pthread_getname_np' function. */ +/* Define to 1 if you have the 'pthread_getname_np' function. */ #undef HAVE_PTHREAD_GETNAME_NP /* Define to 1 if you have the header file. */ #undef HAVE_PTHREAD_H -/* Define to 1 if you have the `pthread_init' function. */ +/* Define to 1 if you have the 'pthread_init' function. */ #undef HAVE_PTHREAD_INIT -/* Define to 1 if you have the `pthread_kill' function. */ +/* Define to 1 if you have the 'pthread_kill' function. */ #undef HAVE_PTHREAD_KILL -/* Define to 1 if you have the `pthread_setname_np' function. */ +/* Define to 1 if you have the 'pthread_setname_np' function. */ #undef HAVE_PTHREAD_SETNAME_NP -/* Define to 1 if you have the `pthread_sigmask' function. */ +/* Define to 1 if you have the 'pthread_sigmask' function. */ #undef HAVE_PTHREAD_SIGMASK /* Define if platform requires stubbed pthreads support */ #undef HAVE_PTHREAD_STUBS -/* Define to 1 if you have the `ptsname' function. */ +/* Define to 1 if you have the 'ptsname' function. */ #undef HAVE_PTSNAME -/* Define to 1 if you have the `ptsname_r' function. */ +/* Define to 1 if you have the 'ptsname_r' function. */ #undef HAVE_PTSNAME_R /* Define to 1 if you have the header file. */ #undef HAVE_PTY_H -/* Define to 1 if you have the `pwrite' function. */ +/* Define to 1 if you have the 'pwrite' function. */ #undef HAVE_PWRITE -/* Define to 1 if you have the `pwritev' function. */ +/* Define to 1 if you have the 'pwritev' function. */ #undef HAVE_PWRITEV -/* Define to 1 if you have the `pwritev2' function. */ +/* Define to 1 if you have the 'pwritev2' function. */ #undef HAVE_PWRITEV2 /* Define to 1 if you have the header file. */ #undef HAVE_READLINE_READLINE_H -/* Define to 1 if you have the `readlink' function. */ +/* Define to 1 if you have the 'readlink' function. */ #undef HAVE_READLINK -/* Define to 1 if you have the `readlinkat' function. */ +/* Define to 1 if you have the 'readlinkat' function. */ #undef HAVE_READLINKAT -/* Define to 1 if you have the `readv' function. */ +/* Define to 1 if you have the 'readv' function. */ #undef HAVE_READV -/* Define to 1 if you have the `realpath' function. */ +/* Define to 1 if you have the 'realpath' function. */ #undef HAVE_REALPATH /* Define if you have the 'recvfrom' function. */ #undef HAVE_RECVFROM -/* Define to 1 if you have the `renameat' function. */ +/* Define to 1 if you have the 'renameat' function. */ #undef HAVE_RENAMEAT /* Define if readline supports append_history */ @@ -1050,7 +1050,7 @@ /* Define if you can turn off readline's signal handling. */ #undef HAVE_RL_CATCH_SIGNAL -/* Define to 1 if the system has the type `rl_compdisp_func_t'. */ +/* Define to 1 if the system has the type 'rl_compdisp_func_t'. */ #undef HAVE_RL_COMPDISP_FUNC_T /* Define if you have readline 2.2 */ @@ -1071,154 +1071,154 @@ /* Define if you have readline 4.0 */ #undef HAVE_RL_RESIZE_TERMINAL -/* Define to 1 if you have the `rtpSpawn' function. */ +/* Define to 1 if you have the 'rtpSpawn' function. */ #undef HAVE_RTPSPAWN -/* Define to 1 if you have the `sched_get_priority_max' function. */ +/* Define to 1 if you have the 'sched_get_priority_max' function. */ #undef HAVE_SCHED_GET_PRIORITY_MAX /* Define to 1 if you have the header file. */ #undef HAVE_SCHED_H -/* Define to 1 if you have the `sched_rr_get_interval' function. */ +/* Define to 1 if you have the 'sched_rr_get_interval' function. */ #undef HAVE_SCHED_RR_GET_INTERVAL -/* Define to 1 if you have the `sched_setaffinity' function. */ +/* Define to 1 if you have the 'sched_setaffinity' function. */ #undef HAVE_SCHED_SETAFFINITY -/* Define to 1 if you have the `sched_setparam' function. */ +/* Define to 1 if you have the 'sched_setparam' function. */ #undef HAVE_SCHED_SETPARAM -/* Define to 1 if you have the `sched_setscheduler' function. */ +/* Define to 1 if you have the 'sched_setscheduler' function. */ #undef HAVE_SCHED_SETSCHEDULER -/* Define to 1 if you have the `sem_clockwait' function. */ +/* Define to 1 if you have the 'sem_clockwait' function. */ #undef HAVE_SEM_CLOCKWAIT -/* Define to 1 if you have the `sem_getvalue' function. */ +/* Define to 1 if you have the 'sem_getvalue' function. */ #undef HAVE_SEM_GETVALUE -/* Define to 1 if you have the `sem_open' function. */ +/* Define to 1 if you have the 'sem_open' function. */ #undef HAVE_SEM_OPEN -/* Define to 1 if you have the `sem_timedwait' function. */ +/* Define to 1 if you have the 'sem_timedwait' function. */ #undef HAVE_SEM_TIMEDWAIT -/* Define to 1 if you have the `sem_unlink' function. */ +/* Define to 1 if you have the 'sem_unlink' function. */ #undef HAVE_SEM_UNLINK -/* Define to 1 if you have the `sendfile' function. */ +/* Define to 1 if you have the 'sendfile' function. */ #undef HAVE_SENDFILE /* Define if you have the 'sendto' function. */ #undef HAVE_SENDTO -/* Define to 1 if you have the `setegid' function. */ +/* Define to 1 if you have the 'setegid' function. */ #undef HAVE_SETEGID -/* Define to 1 if you have the `seteuid' function. */ +/* Define to 1 if you have the 'seteuid' function. */ #undef HAVE_SETEUID -/* Define to 1 if you have the `setgid' function. */ +/* Define to 1 if you have the 'setgid' function. */ #undef HAVE_SETGID /* Define if you have the 'setgroups' function. */ #undef HAVE_SETGROUPS -/* Define to 1 if you have the `sethostname' function. */ +/* Define to 1 if you have the 'sethostname' function. */ #undef HAVE_SETHOSTNAME -/* Define to 1 if you have the `setitimer' function. */ +/* Define to 1 if you have the 'setitimer' function. */ #undef HAVE_SETITIMER /* Define to 1 if you have the header file. */ #undef HAVE_SETJMP_H -/* Define to 1 if you have the `setlocale' function. */ +/* Define to 1 if you have the 'setlocale' function. */ #undef HAVE_SETLOCALE -/* Define to 1 if you have the `setns' function. */ +/* Define to 1 if you have the 'setns' function. */ #undef HAVE_SETNS -/* Define to 1 if you have the `setpgid' function. */ +/* Define to 1 if you have the 'setpgid' function. */ #undef HAVE_SETPGID -/* Define to 1 if you have the `setpgrp' function. */ +/* Define to 1 if you have the 'setpgrp' function. */ #undef HAVE_SETPGRP -/* Define to 1 if you have the `setpriority' function. */ +/* Define to 1 if you have the 'setpriority' function. */ #undef HAVE_SETPRIORITY -/* Define to 1 if you have the `setregid' function. */ +/* Define to 1 if you have the 'setregid' function. */ #undef HAVE_SETREGID -/* Define to 1 if you have the `setresgid' function. */ +/* Define to 1 if you have the 'setresgid' function. */ #undef HAVE_SETRESGID -/* Define to 1 if you have the `setresuid' function. */ +/* Define to 1 if you have the 'setresuid' function. */ #undef HAVE_SETRESUID -/* Define to 1 if you have the `setreuid' function. */ +/* Define to 1 if you have the 'setreuid' function. */ #undef HAVE_SETREUID -/* Define to 1 if you have the `setsid' function. */ +/* Define to 1 if you have the 'setsid' function. */ #undef HAVE_SETSID /* Define if you have the 'setsockopt' function. */ #undef HAVE_SETSOCKOPT -/* Define to 1 if you have the `setuid' function. */ +/* Define to 1 if you have the 'setuid' function. */ #undef HAVE_SETUID -/* Define to 1 if you have the `setvbuf' function. */ +/* Define to 1 if you have the 'setvbuf' function. */ #undef HAVE_SETVBUF /* Define to 1 if you have the header file. */ #undef HAVE_SHADOW_H -/* Define to 1 if you have the `shm_open' function. */ +/* Define to 1 if you have the 'shm_open' function. */ #undef HAVE_SHM_OPEN -/* Define to 1 if you have the `shm_unlink' function. */ +/* Define to 1 if you have the 'shm_unlink' function. */ #undef HAVE_SHM_UNLINK -/* Define to 1 if you have the `shutdown' function. */ +/* Define to 1 if you have the 'shutdown' function. */ #undef HAVE_SHUTDOWN -/* Define to 1 if you have the `sigaction' function. */ +/* Define to 1 if you have the 'sigaction' function. */ #undef HAVE_SIGACTION -/* Define to 1 if you have the `sigaltstack' function. */ +/* Define to 1 if you have the 'sigaltstack' function. */ #undef HAVE_SIGALTSTACK -/* Define to 1 if you have the `sigfillset' function. */ +/* Define to 1 if you have the 'sigfillset' function. */ #undef HAVE_SIGFILLSET -/* Define to 1 if `si_band' is a member of `siginfo_t'. */ +/* Define to 1 if 'si_band' is a member of 'siginfo_t'. */ #undef HAVE_SIGINFO_T_SI_BAND -/* Define to 1 if you have the `siginterrupt' function. */ +/* Define to 1 if you have the 'siginterrupt' function. */ #undef HAVE_SIGINTERRUPT /* Define to 1 if you have the header file. */ #undef HAVE_SIGNAL_H -/* Define to 1 if you have the `sigpending' function. */ +/* Define to 1 if you have the 'sigpending' function. */ #undef HAVE_SIGPENDING -/* Define to 1 if you have the `sigrelse' function. */ +/* Define to 1 if you have the 'sigrelse' function. */ #undef HAVE_SIGRELSE -/* Define to 1 if you have the `sigtimedwait' function. */ +/* Define to 1 if you have the 'sigtimedwait' function. */ #undef HAVE_SIGTIMEDWAIT -/* Define to 1 if you have the `sigwait' function. */ +/* Define to 1 if you have the 'sigwait' function. */ #undef HAVE_SIGWAIT -/* Define to 1 if you have the `sigwaitinfo' function. */ +/* Define to 1 if you have the 'sigwaitinfo' function. */ #undef HAVE_SIGWAITINFO -/* Define to 1 if you have the `snprintf' function. */ +/* Define to 1 if you have the 'snprintf' function. */ #undef HAVE_SNPRINTF /* struct sockaddr_alg (linux/if_alg.h) */ @@ -1236,19 +1236,19 @@ /* Define if you have the 'socketpair' function. */ #undef HAVE_SOCKETPAIR -/* Define to 1 if the system has the type `socklen_t'. */ +/* Define to 1 if the system has the type 'socklen_t'. */ #undef HAVE_SOCKLEN_T /* Define to 1 if you have the header file. */ #undef HAVE_SPAWN_H -/* Define to 1 if you have the `splice' function. */ +/* Define to 1 if you have the 'splice' function. */ #undef HAVE_SPLICE -/* Define to 1 if the system has the type `ssize_t'. */ +/* Define to 1 if the system has the type 'ssize_t'. */ #undef HAVE_SSIZE_T -/* Define to 1 if you have the `statvfs' function. */ +/* Define to 1 if you have the 'statvfs' function. */ #undef HAVE_STATVFS /* Define if you have struct stat.st_mtim.tv_nsec */ @@ -1269,7 +1269,7 @@ /* Has stdatomic.h with atomic_int and atomic_uintptr_t */ #undef HAVE_STD_ATOMIC -/* Define to 1 if you have the `strftime' function. */ +/* Define to 1 if you have the 'strftime' function. */ #undef HAVE_STRFTIME /* Define to 1 if you have the header file. */ @@ -1278,52 +1278,52 @@ /* Define to 1 if you have the header file. */ #undef HAVE_STRING_H -/* Define to 1 if you have the `strlcpy' function. */ +/* Define to 1 if you have the 'strlcpy' function. */ #undef HAVE_STRLCPY /* Define to 1 if you have the header file. */ #undef HAVE_STROPTS_H -/* Define to 1 if you have the `strsignal' function. */ +/* Define to 1 if you have the 'strsignal' function. */ #undef HAVE_STRSIGNAL -/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +/* Define to 1 if 'pw_gecos' is a member of 'struct passwd'. */ #undef HAVE_STRUCT_PASSWD_PW_GECOS -/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +/* Define to 1 if 'pw_passwd' is a member of 'struct passwd'. */ #undef HAVE_STRUCT_PASSWD_PW_PASSWD -/* Define to 1 if `st_birthtime' is a member of `struct stat'. */ +/* Define to 1 if 'st_birthtime' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_BIRTHTIME -/* Define to 1 if `st_blksize' is a member of `struct stat'. */ +/* Define to 1 if 'st_blksize' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_BLKSIZE -/* Define to 1 if `st_blocks' is a member of `struct stat'. */ +/* Define to 1 if 'st_blocks' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_BLOCKS -/* Define to 1 if `st_flags' is a member of `struct stat'. */ +/* Define to 1 if 'st_flags' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_FLAGS -/* Define to 1 if `st_gen' is a member of `struct stat'. */ +/* Define to 1 if 'st_gen' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_GEN -/* Define to 1 if `st_rdev' is a member of `struct stat'. */ +/* Define to 1 if 'st_rdev' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_RDEV -/* Define to 1 if `tm_zone' is a member of `struct tm'. */ +/* Define to 1 if 'tm_zone' is a member of 'struct tm'. */ #undef HAVE_STRUCT_TM_TM_ZONE /* Define if you have the 'symlink' function. */ #undef HAVE_SYMLINK -/* Define to 1 if you have the `symlinkat' function. */ +/* Define to 1 if you have the 'symlinkat' function. */ #undef HAVE_SYMLINKAT -/* Define to 1 if you have the `sync' function. */ +/* Define to 1 if you have the 'sync' function. */ #undef HAVE_SYNC -/* Define to 1 if you have the `sysconf' function. */ +/* Define to 1 if you have the 'sysconf' function. */ #undef HAVE_SYSCONF /* Define to 1 if you have the header file. */ @@ -1332,7 +1332,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYSLOG_H -/* Define to 1 if you have the `system' function. */ +/* Define to 1 if you have the 'system' function. */ #undef HAVE_SYSTEM /* Define to 1 if you have the header file. */ @@ -1347,7 +1347,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_DEVPOLL_H -/* Define to 1 if you have the header file, and it defines `DIR'. +/* Define to 1 if you have the header file, and it defines 'DIR'. */ #undef HAVE_SYS_DIR_H @@ -1390,7 +1390,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_MODEM_H -/* Define to 1 if you have the header file, and it defines `DIR'. +/* Define to 1 if you have the header file, and it defines 'DIR'. */ #undef HAVE_SYS_NDIR_H @@ -1466,13 +1466,13 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_XATTR_H -/* Define to 1 if you have the `tcgetpgrp' function. */ +/* Define to 1 if you have the 'tcgetpgrp' function. */ #undef HAVE_TCGETPGRP -/* Define to 1 if you have the `tcsetpgrp' function. */ +/* Define to 1 if you have the 'tcsetpgrp' function. */ #undef HAVE_TCSETPGRP -/* Define to 1 if you have the `tempnam' function. */ +/* Define to 1 if you have the 'tempnam' function. */ #undef HAVE_TEMPNAM /* Define to 1 if you have the header file. */ @@ -1481,54 +1481,54 @@ /* Define to 1 if you have the header file. */ #undef HAVE_TERM_H -/* Define to 1 if you have the `timegm' function. */ +/* Define to 1 if you have the 'timegm' function. */ #undef HAVE_TIMEGM /* Define if you have the 'timerfd_create' function. */ #undef HAVE_TIMERFD_CREATE -/* Define to 1 if you have the `times' function. */ +/* Define to 1 if you have the 'times' function. */ #undef HAVE_TIMES -/* Define to 1 if you have the `tmpfile' function. */ +/* Define to 1 if you have the 'tmpfile' function. */ #undef HAVE_TMPFILE -/* Define to 1 if you have the `tmpnam' function. */ +/* Define to 1 if you have the 'tmpnam' function. */ #undef HAVE_TMPNAM -/* Define to 1 if you have the `tmpnam_r' function. */ +/* Define to 1 if you have the 'tmpnam_r' function. */ #undef HAVE_TMPNAM_R -/* Define to 1 if your `struct tm' has `tm_zone'. Deprecated, use - `HAVE_STRUCT_TM_TM_ZONE' instead. */ +/* Define to 1 if your 'struct tm' has 'tm_zone'. Deprecated, use + 'HAVE_STRUCT_TM_TM_ZONE' instead. */ #undef HAVE_TM_ZONE -/* Define to 1 if you have the `truncate' function. */ +/* Define to 1 if you have the 'truncate' function. */ #undef HAVE_TRUNCATE -/* Define to 1 if you have the `ttyname' function. */ +/* Define to 1 if you have the 'ttyname' function. */ #undef HAVE_TTYNAME -/* Define to 1 if you don't have `tm_zone' but do have the external array - `tzname'. */ +/* Define to 1 if you don't have 'tm_zone' but do have the external array + 'tzname'. */ #undef HAVE_TZNAME -/* Define to 1 if you have the `umask' function. */ +/* Define to 1 if you have the 'umask' function. */ #undef HAVE_UMASK -/* Define to 1 if you have the `uname' function. */ +/* Define to 1 if you have the 'uname' function. */ #undef HAVE_UNAME /* Define to 1 if you have the header file. */ #undef HAVE_UNISTD_H -/* Define to 1 if you have the `unlinkat' function. */ +/* Define to 1 if you have the 'unlinkat' function. */ #undef HAVE_UNLINKAT -/* Define to 1 if you have the `unlockpt' function. */ +/* Define to 1 if you have the 'unlockpt' function. */ #undef HAVE_UNLOCKPT -/* Define to 1 if you have the `unshare' function. */ +/* Define to 1 if you have the 'unshare' function. */ #undef HAVE_UNSHARE /* Define if you have a useable wchar_t type defined in wchar.h; useable means @@ -1539,10 +1539,10 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UTIL_H -/* Define to 1 if you have the `utimensat' function. */ +/* Define to 1 if you have the 'utimensat' function. */ #undef HAVE_UTIMENSAT -/* Define to 1 if you have the `utimes' function. */ +/* Define to 1 if you have the 'utimes' function. */ #undef HAVE_UTIMES /* Define to 1 if you have the header file. */ @@ -1551,10 +1551,10 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UTMP_H -/* Define to 1 if you have the `uuid_create' function. */ +/* Define to 1 if you have the 'uuid_create' function. */ #undef HAVE_UUID_CREATE -/* Define to 1 if you have the `uuid_enc_be' function. */ +/* Define to 1 if you have the 'uuid_enc_be' function. */ #undef HAVE_UUID_ENC_BE /* Define if uuid_generate_time_safe() exists. */ @@ -1566,44 +1566,44 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UUID_UUID_H -/* Define to 1 if you have the `vfork' function. */ +/* Define to 1 if you have the 'vfork' function. */ #undef HAVE_VFORK -/* Define to 1 if you have the `wait' function. */ +/* Define to 1 if you have the 'wait' function. */ #undef HAVE_WAIT -/* Define to 1 if you have the `wait3' function. */ +/* Define to 1 if you have the 'wait3' function. */ #undef HAVE_WAIT3 -/* Define to 1 if you have the `wait4' function. */ +/* Define to 1 if you have the 'wait4' function. */ #undef HAVE_WAIT4 -/* Define to 1 if you have the `waitid' function. */ +/* Define to 1 if you have the 'waitid' function. */ #undef HAVE_WAITID -/* Define to 1 if you have the `waitpid' function. */ +/* Define to 1 if you have the 'waitpid' function. */ #undef HAVE_WAITPID /* Define if the compiler provides a wchar.h header file. */ #undef HAVE_WCHAR_H -/* Define to 1 if you have the `wcscoll' function. */ +/* Define to 1 if you have the 'wcscoll' function. */ #undef HAVE_WCSCOLL -/* Define to 1 if you have the `wcsftime' function. */ +/* Define to 1 if you have the 'wcsftime' function. */ #undef HAVE_WCSFTIME -/* Define to 1 if you have the `wcsxfrm' function. */ +/* Define to 1 if you have the 'wcsxfrm' function. */ #undef HAVE_WCSXFRM -/* Define to 1 if you have the `wmemcmp' function. */ +/* Define to 1 if you have the 'wmemcmp' function. */ #undef HAVE_WMEMCMP /* Define if tzset() actually switches the local timezone in a meaningful way. */ #undef HAVE_WORKING_TZSET -/* Define to 1 if you have the `writev' function. */ +/* Define to 1 if you have the 'writev' function. */ #undef HAVE_WRITEV /* Define if the zlib library has inflateCopy */ @@ -1612,17 +1612,17 @@ /* Define to 1 if you have the header file. */ #undef HAVE_ZLIB_H -/* Define to 1 if you have the `_getpty' function. */ +/* Define to 1 if you have the '_getpty' function. */ #undef HAVE__GETPTY -/* Define to 1 if the system has the type `__uint128_t'. */ +/* Define to 1 if the system has the type '__uint128_t'. */ #undef HAVE___UINT128_T -/* Define to 1 if `major', `minor', and `makedev' are declared in . +/* Define to 1 if 'major', 'minor', and 'makedev' are declared in . */ #undef MAJOR_IN_MKDEV -/* Define to 1 if `major', `minor', and `makedev' are declared in +/* Define to 1 if 'major', 'minor', and 'makedev' are declared in . */ #undef MAJOR_IN_SYSMACROS @@ -1730,58 +1730,58 @@ /* Define if i>>j for signed int i does not extend the sign bit when i < 0 */ #undef SIGNED_RIGHT_SHIFT_ZERO_FILLS -/* The size of `double', as computed by sizeof. */ +/* The size of 'double', as computed by sizeof. */ #undef SIZEOF_DOUBLE -/* The size of `float', as computed by sizeof. */ +/* The size of 'float', as computed by sizeof. */ #undef SIZEOF_FLOAT -/* The size of `fpos_t', as computed by sizeof. */ +/* The size of 'fpos_t', as computed by sizeof. */ #undef SIZEOF_FPOS_T -/* The size of `int', as computed by sizeof. */ +/* The size of 'int', as computed by sizeof. */ #undef SIZEOF_INT -/* The size of `long', as computed by sizeof. */ +/* The size of 'long', as computed by sizeof. */ #undef SIZEOF_LONG -/* The size of `long double', as computed by sizeof. */ +/* The size of 'long double', as computed by sizeof. */ #undef SIZEOF_LONG_DOUBLE -/* The size of `long long', as computed by sizeof. */ +/* The size of 'long long', as computed by sizeof. */ #undef SIZEOF_LONG_LONG -/* The size of `off_t', as computed by sizeof. */ +/* The size of 'off_t', as computed by sizeof. */ #undef SIZEOF_OFF_T -/* The size of `pid_t', as computed by sizeof. */ +/* The size of 'pid_t', as computed by sizeof. */ #undef SIZEOF_PID_T -/* The size of `pthread_key_t', as computed by sizeof. */ +/* The size of 'pthread_key_t', as computed by sizeof. */ #undef SIZEOF_PTHREAD_KEY_T -/* The size of `pthread_t', as computed by sizeof. */ +/* The size of 'pthread_t', as computed by sizeof. */ #undef SIZEOF_PTHREAD_T -/* The size of `short', as computed by sizeof. */ +/* The size of 'short', as computed by sizeof. */ #undef SIZEOF_SHORT -/* The size of `size_t', as computed by sizeof. */ +/* The size of 'size_t', as computed by sizeof. */ #undef SIZEOF_SIZE_T -/* The size of `time_t', as computed by sizeof. */ +/* The size of 'time_t', as computed by sizeof. */ #undef SIZEOF_TIME_T -/* The size of `uintptr_t', as computed by sizeof. */ +/* The size of 'uintptr_t', as computed by sizeof. */ #undef SIZEOF_UINTPTR_T -/* The size of `void *', as computed by sizeof. */ +/* The size of 'void *', as computed by sizeof. */ #undef SIZEOF_VOID_P -/* The size of `wchar_t', as computed by sizeof. */ +/* The size of 'wchar_t', as computed by sizeof. */ #undef SIZEOF_WCHAR_T -/* The size of `_Bool', as computed by sizeof. */ +/* The size of '_Bool', as computed by sizeof. */ #undef SIZEOF__BOOL /* Define to 1 if you have the ANSI C header files. */ @@ -1797,13 +1797,13 @@ /* Library needed by timemodule.c: librt may be needed for clock_gettime() */ #undef TIMEMODULE_LIB -/* Define to 1 if your declares `struct tm'. */ +/* Define to 1 if your declares 'struct tm'. */ #undef TM_IN_SYS_TIME /* Define if you want to use computed gotos in ceval.c. */ #undef USE_COMPUTED_GOTOS -/* Enable extensions on AIX 3, Interix. */ +/* Enable extensions on AIX, Interix, z/OS. */ #ifndef _ALL_SOURCE # undef _ALL_SOURCE #endif @@ -1864,11 +1864,15 @@ #ifndef __STDC_WANT_IEC_60559_DFP_EXT__ # undef __STDC_WANT_IEC_60559_DFP_EXT__ #endif +/* Enable extensions specified by C23 Annex F. */ +#ifndef __STDC_WANT_IEC_60559_EXT__ +# undef __STDC_WANT_IEC_60559_EXT__ +#endif /* Enable extensions specified by ISO/IEC TS 18661-4:2015. */ #ifndef __STDC_WANT_IEC_60559_FUNCS_EXT__ # undef __STDC_WANT_IEC_60559_FUNCS_EXT__ #endif -/* Enable extensions specified by ISO/IEC TS 18661-3:2015. */ +/* Enable extensions specified by C23 Annex H and ISO/IEC TS 18661-3:2015. */ #ifndef __STDC_WANT_IEC_60559_TYPES_EXT__ # undef __STDC_WANT_IEC_60559_TYPES_EXT__ #endif @@ -1997,16 +2001,16 @@ /* Define to 'long' if does not define clock_t. */ #undef clock_t -/* Define to empty if `const' does not conform to ANSI C. */ +/* Define to empty if 'const' does not conform to ANSI C. */ #undef const -/* Define to `int' if doesn't define. */ +/* Define as 'int' if doesn't define. */ #undef gid_t -/* Define to `int' if does not define. */ +/* Define to 'int' if does not define. */ #undef mode_t -/* Define to `long int' if does not define. */ +/* Define to 'long int' if does not define. */ #undef off_t /* Define as a signed integer type capable of holding a process identifier. */ @@ -2015,13 +2019,13 @@ /* Define to empty if the keyword does not work. */ #undef signed -/* Define to `unsigned int' if does not define. */ +/* Define as 'unsigned int' if doesn't define. */ #undef size_t /* Define to 'int' if does not define. */ #undef socklen_t -/* Define to `int' if doesn't define. */ +/* Define as 'int' if doesn't define. */ #undef uid_t From fa985bee6189aabac1c329f2de32aa9a4e88e550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 3 Jan 2025 13:37:02 +0100 Subject: [PATCH 347/427] gh-127787: refactor helpers for `PyUnicodeErrorObject` internal interface (GH-127789) - Unify `get_unicode` and `get_string` in a single function. - Allow to retrieve the underlying `object` attribute, its size, and the adjusted 'start' and 'end', all at once. Add a new `_PyUnicodeError_GetParams` internal function for this. (In `exceptions.c`, it's somewhat common to not need all the attributes, but the compiler has opportunity to inline the function and optimize unneeded work away. Outside that file, we'll usually need all or most of them at once.) - Use a common implementation for the following functions: - `PyUnicode{Decode,Encode}Error_GetEncoding` - `PyUnicode{Decode,Encode,Translate}Error_GetObject` - `PyUnicode{Decode,Encode,Translate}Error_{Get,Set}Reason` - `PyUnicode{Decode,Encode,Translate}Error_{Get,Set}{Start,End}` --- Include/cpython/pyerrors.h | 6 + Objects/exceptions.c | 442 ++++++++++++++++++++++++------------- 2 files changed, 293 insertions(+), 155 deletions(-) diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index b36b4681f5dddb..49a6265e5eb02f 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -94,6 +94,12 @@ PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); /* In exceptions.c */ +PyAPI_FUNC(int) _PyUnicodeError_GetParams( + PyObject *self, + PyObject **obj, Py_ssize_t *objlen, + Py_ssize_t *start, Py_ssize_t *end, + int as_bytes); + PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( PyObject *orig, PyObject *excs); diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 6880c24196cbb8..714f8c828afbc1 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2667,47 +2667,167 @@ SimpleExtendsException(PyExc_Exception, ValueError, SimpleExtendsException(PyExc_ValueError, UnicodeError, "Unicode related error."); + +/* + * Check the validity of 'attr' as a unicode or bytes object depending + * on 'as_bytes' and return a new reference on it if it is the case. + * + * The 'name' is the attribute name and is only used for error reporting. + * + * On success, this returns a strong reference on 'attr'. + * On failure, this sets a TypeError and returns NULL. + */ static PyObject * -get_bytes(PyObject *attr, const char *name) +as_unicode_error_attribute(PyObject *attr, const char *name, int as_bytes) { - if (!attr) { - PyErr_Format(PyExc_TypeError, "%.200s attribute not set", name); + assert(as_bytes == 0 || as_bytes == 1); + if (attr == NULL) { + PyErr_Format(PyExc_TypeError, "%s attribute not set", name); return NULL; } - - if (!PyBytes_Check(attr)) { - PyErr_Format(PyExc_TypeError, "%.200s attribute must be bytes", name); + if (!(as_bytes ? PyBytes_Check(attr) : PyUnicode_Check(attr))) { + PyErr_Format(PyExc_TypeError, + "%s attribute must be %s", + name, + as_bytes ? "bytes" : "unicode"); return NULL; } return Py_NewRef(attr); } -static PyObject * -get_unicode(PyObject *attr, const char *name) -{ - if (!attr) { - PyErr_Format(PyExc_TypeError, "%.200s attribute not set", name); - return NULL; - } - if (!PyUnicode_Check(attr)) { +#define PyUnicodeError_Check(PTR) \ + PyObject_TypeCheck((PTR), (PyTypeObject *)PyExc_UnicodeError) +#define PyUnicodeError_CAST(PTR) \ + (assert(PyUnicodeError_Check(PTR)), ((PyUnicodeErrorObject *)(PTR))) + + +/* class names to use when reporting errors */ +#define Py_UNICODE_ENCODE_ERROR_NAME "UnicodeEncodeError" +#define Py_UNICODE_DECODE_ERROR_NAME "UnicodeDecodeError" +#define Py_UNICODE_TRANSLATE_ERROR_NAME "UnicodeTranslateError" + + +/* + * Check that 'self' is a UnicodeError object. + * + * On success, this returns 0. + * On failure, this sets a TypeError exception and returns -1. + * + * The 'expect_type' is the name of the expected type, which is + * only used for error reporting. + * + * As an implementation detail, the `PyUnicode*Error_*` functions + * currently allow *any* subclass of UnicodeError as 'self'. + * + * Use one of the `Py_UNICODE_*_ERROR_NAME` macros to avoid typos. + */ +static inline int +check_unicode_error_type(PyObject *self, const char *expect_type) +{ + assert(self != NULL); + if (!PyUnicodeError_Check(self)) { PyErr_Format(PyExc_TypeError, - "%.200s attribute must be unicode", name); - return NULL; + "expecting a %s object, got %T", expect_type, self); + return -1; } - return Py_NewRef(attr); + return 0; } -static int -set_unicodefromstring(PyObject **attr, const char *value) + +// --- PyUnicodeEncodeObject: internal helpers -------------------------------- +// +// In the helpers below, the caller is responsible to ensure that 'self' +// is a PyUnicodeErrorObject, although this is verified on DEBUG builds +// through PyUnicodeError_CAST(). + +/* + * Return the underlying (str) 'encoding' attribute of a UnicodeError object. + */ +static inline PyObject * +unicode_error_get_encoding_impl(PyObject *self) +{ + assert(self != NULL); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + return as_unicode_error_attribute(exc->encoding, "encoding", false); +} + + +/* + * Return the underlying 'object' attribute of a UnicodeError object + * as a bytes or a string instance, depending on the 'as_bytes' flag. + */ +static inline PyObject * +unicode_error_get_object_impl(PyObject *self, int as_bytes) { - PyObject *obj = PyUnicode_FromString(value); - if (!obj) + assert(self != NULL); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + return as_unicode_error_attribute(exc->object, "object", as_bytes); +} + + +/* + * Return the underlying (str) 'reason' attribute of a UnicodeError object. + */ +static inline PyObject * +unicode_error_get_reason_impl(PyObject *self) +{ + assert(self != NULL); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + return as_unicode_error_attribute(exc->reason, "reason", false); +} + + +/* + * Set the underlying (str) 'reason' attribute of a UnicodeError object. + * + * Return 0 on success and -1 on failure. + */ +static inline int +unicode_error_set_reason_impl(PyObject *self, const char *reason) +{ + assert(self != NULL); + PyObject *value = PyUnicode_FromString(reason); + if (value == NULL) { return -1; - Py_XSETREF(*attr, obj); + } + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + Py_XSETREF(exc->reason, value); return 0; } + +/* + * Set the 'start' attribute of a UnicodeError object. + * + * Return 0 on success and -1 on failure. + */ +static inline int +unicode_error_set_start_impl(PyObject *self, Py_ssize_t start) +{ + assert(self != NULL); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + exc->start = start; + return 0; +} + + +/* + * Set the 'end' attribute of a UnicodeError object. + * + * Return 0 on success and -1 on failure. + */ +static inline int +unicode_error_set_end_impl(PyObject *self, Py_ssize_t end) +{ + assert(self != NULL); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + exc->end = end; + return 0; +} + +// --- PyUnicodeEncodeObject: internal getters -------------------------------- + /* * Adjust the (inclusive) 'start' value of a UnicodeError object. * @@ -2728,6 +2848,7 @@ unicode_error_adjust_start(Py_ssize_t start, Py_ssize_t objlen) return start; } + /* * Adjust the (exclusive) 'end' value of a UnicodeError object. * @@ -2748,134 +2869,162 @@ unicode_error_adjust_end(Py_ssize_t end, Py_ssize_t objlen) return end; } -#define _PyUnicodeError_CAST(PTR) ((PyUnicodeErrorObject *)(PTR)) -#define PyUnicodeError_Check(PTR) \ - PyObject_TypeCheck((PTR), (PyTypeObject *)PyExc_UnicodeError) -#define PyUnicodeError_CAST(PTR) \ - (assert(PyUnicodeError_Check(PTR)), _PyUnicodeError_CAST(PTR)) - -static inline int -check_unicode_error_type(PyObject *self, const char *expect_type) +/* + * Get various common parameters of a UnicodeError object. + * + * The caller is responsible to ensure that 'self' is a PyUnicodeErrorObject, + * although this condition is verified by this function on DEBUG builds. + * + * Return 0 on success and -1 on failure. + * + * Output parameters: + * + * obj A strong reference to the 'object' attribute. + * objlen The 'object' length. + * start The clipped 'start' attribute. + * end The clipped 'end' attribute. + * + * An output parameter can be NULL to indicate that + * the corresponding value does not need to be stored. + * + * Input parameter: + * + * as_bytes If 1, the error's 'object' attribute must be a bytes object, + * i.e. the call is for a `UnicodeDecodeError`. Otherwise, the + * 'object' attribute must be a string. + * + * A TypeError is raised if the 'object' type is incompatible. + */ +int +_PyUnicodeError_GetParams(PyObject *self, + PyObject **obj, Py_ssize_t *objlen, + Py_ssize_t *start, Py_ssize_t *end, + int as_bytes) { - if (!PyUnicodeError_Check(self)) { - PyErr_Format(PyExc_TypeError, - "expecting a %s object, got %T", expect_type, self); + assert(self != NULL); + assert(as_bytes == 0 || as_bytes == 1); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + PyObject *r = as_unicode_error_attribute(exc->object, "object", as_bytes); + if (r == NULL) { return -1; } + + Py_ssize_t n = as_bytes ? PyBytes_GET_SIZE(r) : PyUnicode_GET_LENGTH(r); + if (objlen != NULL) { + *objlen = n; + } + if (start != NULL) { + *start = unicode_error_adjust_start(exc->start, n); + assert(*start >= 0); + assert(*start <= n); + } + if (end != NULL) { + *end = unicode_error_adjust_end(exc->end, n); + assert(*end >= 0); + assert(*end <= n); + } + if (obj != NULL) { + *obj = r; + } + else { + Py_DECREF(r); + } return 0; } -static inline PyUnicodeErrorObject * -as_unicode_error(PyObject *self, const char *expect_type) -{ - int rc = check_unicode_error_type(self, expect_type); - return rc < 0 ? NULL : _PyUnicodeError_CAST(self); -} +// --- PyUnicodeEncodeObject: 'encoding' getters ------------------------------ +// Note: PyUnicodeTranslateError does not have an 'encoding' attribute. PyObject * PyUnicodeEncodeError_GetEncoding(PyObject *self) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); - return exc == NULL ? NULL : get_unicode(exc->encoding, "encoding"); + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_encoding_impl(self); } + PyObject * PyUnicodeDecodeError_GetEncoding(PyObject *self) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); - return exc == NULL ? NULL : get_unicode(exc->encoding, "encoding"); + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_encoding_impl(self); } + +// --- PyUnicodeEncodeObject: 'object' getters -------------------------------- + PyObject * PyUnicodeEncodeError_GetObject(PyObject *self) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); - return exc == NULL ? NULL : get_unicode(exc->object, "object"); + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_object_impl(self, false); } + PyObject * PyUnicodeDecodeError_GetObject(PyObject *self) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); - return exc == NULL ? NULL : get_bytes(exc->object, "object"); + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_object_impl(self, true); } + PyObject * PyUnicodeTranslateError_GetObject(PyObject *self) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeTranslateError"); - return exc == NULL ? NULL : get_unicode(exc->object, "object"); + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_object_impl(self, false); } + +// --- PyUnicodeEncodeObject: 'start' getters --------------------------------- + +/* + * Specialization of _PyUnicodeError_GetParams() for the 'start' attribute. + * + * The caller is responsible to ensure that 'self' is a PyUnicodeErrorObject, + * although this condition is verified by this function on DEBUG builds. + */ +static inline int +unicode_error_get_start_impl(PyObject *self, Py_ssize_t *start, int as_bytes) +{ + assert(self != NULL); + return _PyUnicodeError_GetParams(self, NULL, NULL, start, NULL, as_bytes); +} + + int PyUnicodeEncodeError_GetStart(PyObject *self, Py_ssize_t *start) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); - if (exc == NULL) { - return -1; - } - PyObject *obj = get_unicode(exc->object, "object"); - if (obj == NULL) { - return -1; - } - Py_ssize_t size = PyUnicode_GET_LENGTH(obj); - Py_DECREF(obj); - *start = unicode_error_adjust_start(exc->start, size); - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_start_impl(self, start, false); } int PyUnicodeDecodeError_GetStart(PyObject *self, Py_ssize_t *start) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); - if (exc == NULL) { - return -1; - } - PyObject *obj = get_bytes(exc->object, "object"); - if (obj == NULL) { - return -1; - } - Py_ssize_t size = PyBytes_GET_SIZE(obj); - Py_DECREF(obj); - *start = unicode_error_adjust_start(exc->start, size); - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_start_impl(self, start, true); } int PyUnicodeTranslateError_GetStart(PyObject *self, Py_ssize_t *start) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeTranslateError"); - if (exc == NULL) { - return -1; - } - PyObject *obj = get_unicode(exc->object, "object"); - if (obj == NULL) { - return -1; - } - Py_ssize_t size = PyUnicode_GET_LENGTH(obj); - Py_DECREF(obj); - *start = unicode_error_adjust_start(exc->start, size); - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_start_impl(self, start, false); } -static inline int -unicode_error_set_start_impl(PyObject *self, Py_ssize_t start) -{ - PyUnicodeErrorObject *exc = _PyUnicodeError_CAST(self); - exc->start = start; - return 0; -} - +// --- PyUnicodeEncodeObject: 'start' setters --------------------------------- int PyUnicodeEncodeError_SetStart(PyObject *self, Py_ssize_t start) { - int rc = check_unicode_error_type(self, "UnicodeEncodeError"); + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); return rc < 0 ? -1 : unicode_error_set_start_impl(self, start); } @@ -2883,7 +3032,7 @@ PyUnicodeEncodeError_SetStart(PyObject *self, Py_ssize_t start) int PyUnicodeDecodeError_SetStart(PyObject *self, Py_ssize_t start) { - int rc = check_unicode_error_type(self, "UnicodeDecodeError"); + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); return rc < 0 ? -1 : unicode_error_set_start_impl(self, start); } @@ -2891,78 +3040,57 @@ PyUnicodeDecodeError_SetStart(PyObject *self, Py_ssize_t start) int PyUnicodeTranslateError_SetStart(PyObject *self, Py_ssize_t start) { - int rc = check_unicode_error_type(self, "UnicodeTranslateError"); + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); return rc < 0 ? -1 : unicode_error_set_start_impl(self, start); } +// --- PyUnicodeEncodeObject: 'end' getters ----------------------------------- + +/* + * Specialization of _PyUnicodeError_GetParams() for the 'end' attribute. + * + * The caller is responsible to ensure that 'self' is a PyUnicodeErrorObject, + * although this condition is verified by this function on DEBUG builds. + */ +static inline int +unicode_error_get_end_impl(PyObject *self, Py_ssize_t *end, int as_bytes) +{ + assert(self != NULL); + return _PyUnicodeError_GetParams(self, NULL, NULL, NULL, end, as_bytes); +} + + int PyUnicodeEncodeError_GetEnd(PyObject *self, Py_ssize_t *end) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); - if (exc == NULL) { - return -1; - } - PyObject *obj = get_unicode(exc->object, "object"); - if (obj == NULL) { - return -1; - } - Py_ssize_t size = PyUnicode_GET_LENGTH(obj); - Py_DECREF(obj); - *end = unicode_error_adjust_end(exc->end, size); - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_end_impl(self, end, false); } int PyUnicodeDecodeError_GetEnd(PyObject *self, Py_ssize_t *end) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); - if (exc == NULL) { - return -1; - } - PyObject *obj = get_bytes(exc->object, "object"); - if (obj == NULL) { - return -1; - } - Py_ssize_t size = PyBytes_GET_SIZE(obj); - Py_DECREF(obj); - *end = unicode_error_adjust_end(exc->end, size); - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_end_impl(self, end, true); } int PyUnicodeTranslateError_GetEnd(PyObject *self, Py_ssize_t *end) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeTranslateError"); - if (exc == NULL) { - return -1; - } - PyObject *obj = get_unicode(exc->object, "object"); - if (obj == NULL) { - return -1; - } - Py_ssize_t size = PyUnicode_GET_LENGTH(obj); - Py_DECREF(obj); - *end = unicode_error_adjust_end(exc->end, size); - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_end_impl(self, end, false); } -static inline int -unicode_error_set_end_impl(PyObject *self, Py_ssize_t end) -{ - PyUnicodeErrorObject *exc = _PyUnicodeError_CAST(self); - exc->end = end; - return 0; -} - +// --- PyUnicodeEncodeObject: 'end' setters ----------------------------------- int PyUnicodeEncodeError_SetEnd(PyObject *self, Py_ssize_t end) { - int rc = check_unicode_error_type(self, "UnicodeEncodeError"); + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); return rc < 0 ? -1 : unicode_error_set_end_impl(self, end); } @@ -2970,7 +3098,7 @@ PyUnicodeEncodeError_SetEnd(PyObject *self, Py_ssize_t end) int PyUnicodeDecodeError_SetEnd(PyObject *self, Py_ssize_t end) { - int rc = check_unicode_error_type(self, "UnicodeDecodeError"); + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); return rc < 0 ? -1 : unicode_error_set_end_impl(self, end); } @@ -2978,56 +3106,60 @@ PyUnicodeDecodeError_SetEnd(PyObject *self, Py_ssize_t end) int PyUnicodeTranslateError_SetEnd(PyObject *self, Py_ssize_t end) { - int rc = check_unicode_error_type(self, "UnicodeTranslateError"); + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); return rc < 0 ? -1 : unicode_error_set_end_impl(self, end); } +// --- PyUnicodeEncodeObject: 'reason' getters -------------------------------- + PyObject * PyUnicodeEncodeError_GetReason(PyObject *self) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); - return exc == NULL ? NULL : get_unicode(exc->reason, "reason"); + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_reason_impl(self); } PyObject * PyUnicodeDecodeError_GetReason(PyObject *self) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); - return exc == NULL ? NULL : get_unicode(exc->reason, "reason"); + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_reason_impl(self); } PyObject * PyUnicodeTranslateError_GetReason(PyObject *self) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeTranslateError"); - return exc == NULL ? NULL : get_unicode(exc->reason, "reason"); + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_reason_impl(self); } +// --- PyUnicodeEncodeObject: 'reason' setters -------------------------------- + int PyUnicodeEncodeError_SetReason(PyObject *self, const char *reason) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeEncodeError"); - return exc == NULL ? -1 : set_unicodefromstring(&exc->reason, reason); + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_reason_impl(self, reason); } int PyUnicodeDecodeError_SetReason(PyObject *self, const char *reason) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeDecodeError"); - return exc == NULL ? -1 : set_unicodefromstring(&exc->reason, reason); + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_reason_impl(self, reason); } int PyUnicodeTranslateError_SetReason(PyObject *self, const char *reason) { - PyUnicodeErrorObject *exc = as_unicode_error(self, "UnicodeTranslateError"); - return exc == NULL ? -1 : set_unicodefromstring(&exc->reason, reason); + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_reason_impl(self, reason); } From f21af186bf21c1c554209ac67d78d3cf99f7d7c0 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 3 Jan 2025 14:56:24 +0200 Subject: [PATCH 348/427] gh-128317: Highlight today in colour in calendar CLI output (#128318) Co-authored-by: Peter Bierma --- Doc/library/calendar.rst | 31 ++++++++++-- Doc/whatsnew/3.14.rst | 13 +++++ Lib/_colorize.py | 2 + Lib/calendar.py | 50 +++++++++++++++---- ...-12-29-00-33-34.gh-issue-128317.WgFina.rst | 2 + 5 files changed, 85 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-29-00-33-34.gh-issue-128317.WgFina.rst diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 97ca34b6c6184c..ace8529d6e7e0c 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -146,26 +146,34 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is the specified width, representing an empty day. The *weekday* parameter is unused. - .. method:: formatweek(theweek, w=0) + .. method:: formatweek(theweek, w=0, highlight_day=None) Return a single week in a string with no newline. If *w* is provided, it specifies the width of the date columns, which are centered. Depends on the first weekday as specified in the constructor or set by the :meth:`setfirstweekday` method. + .. versionchanged:: next + If *highlight_day* is given, this date is highlighted in color. + This can be :ref:`controlled using environment variables + `. + + .. method:: formatweekday(weekday, width) Return a string representing the name of a single weekday formatted to the specified *width*. The *weekday* parameter is an integer representing the day of the week, where ``0`` is Monday and ``6`` is Sunday. + .. method:: formatweekheader(width) Return a string containing the header row of weekday names, formatted with the given *width* for each column. The names depend on the locale settings and are padded to the specified width. - .. method:: formatmonth(theyear, themonth, w=0, l=0) + + .. method:: formatmonth(theyear, themonth, w=0, l=0, highlight_day=None) Return a month's calendar in a multi-line string. If *w* is provided, it specifies the width of the date columns, which are centered. If *l* is @@ -173,6 +181,12 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is on the first weekday as specified in the constructor or set by the :meth:`setfirstweekday` method. + .. versionchanged:: next + If *highlight_day* is given, this date is highlighted in color. + This can be :ref:`controlled using environment variables + `. + + .. method:: formatmonthname(theyear, themonth, width=0, withyear=True) Return a string representing the month's name centered within the @@ -180,12 +194,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is output. The *theyear* and *themonth* parameters specify the year and month for the name to be formatted respectively. + .. method:: prmonth(theyear, themonth, w=0, l=0) Print a month's calendar as returned by :meth:`formatmonth`. - .. method:: formatyear(theyear, w=2, l=1, c=6, m=3) + .. method:: formatyear(theyear, w=2, l=1, c=6, m=3, highlight_day=None) Return a *m*-column calendar for an entire year as a multi-line string. Optional parameters *w*, *l*, and *c* are for date column width, lines per @@ -194,6 +209,11 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :meth:`setfirstweekday` method. The earliest year for which a calendar can be generated is platform-dependent. + .. versionchanged:: next + If *highlight_day* is given, this date is highlighted in color. + This can be :ref:`controlled using environment variables + `. + .. method:: pryear(theyear, w=2, l=1, c=6, m=3) @@ -549,7 +569,7 @@ The :mod:`calendar` module defines the following exceptions: .. _calendar-cli: -Command-Line Usage +Command-line usage ------------------ .. versionadded:: 2.5 @@ -687,6 +707,9 @@ The following options are accepted: The number of months printed per row. Defaults to 3. +.. versionchanged:: next + By default, today's date is highlighted in color and can be + :ref:`controlled using environment variables `. *HTML-mode options:* diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 69c356ba60e6d4..cb9167300260cb 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -296,6 +296,19 @@ ast * The ``repr()`` output for AST nodes now includes more information. (Contributed by Tomas R in :gh:`116022`.) + +calendar +-------- + +* By default, today's date is highlighted in color in :mod:`calendar`'s + :ref:`command-line ` text output. + This can be controlled via the :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ + and |FORCE_COLOR|_ environment variables. + See also :ref:`using-on-controlling-color`. + (Contributed by Hugo van Kemenade in :gh:`128317`.) + + concurrent.futures ------------------ diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 709081e25ec59b..f609901887a26b 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -6,9 +6,11 @@ class ANSIColors: + BACKGROUND_YELLOW = "\x1b[43m" BOLD_GREEN = "\x1b[1;32m" BOLD_MAGENTA = "\x1b[1;35m" BOLD_RED = "\x1b[1;31m" + BLACK = "\x1b[30m" GREEN = "\x1b[32m" GREY = "\x1b[90m" MAGENTA = "\x1b[35m" diff --git a/Lib/calendar.py b/Lib/calendar.py index 8c1c646da46a98..d2e5e08d02dbb9 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -349,11 +349,27 @@ def formatday(self, day, weekday, width): s = '%2i' % day # right-align single-digit days return s.center(width) - def formatweek(self, theweek, width): + def formatweek(self, theweek, width, *, highlight_day=None): """ Returns a single week in a string (no newline). """ - return ' '.join(self.formatday(d, wd, width) for (d, wd) in theweek) + if highlight_day: + from _colorize import get_colors + + ansi = get_colors() + highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" + reset = ansi.RESET + else: + highlight = reset = "" + + return ' '.join( + ( + f"{highlight}{self.formatday(d, wd, width)}{reset}" + if d == highlight_day + else self.formatday(d, wd, width) + ) + for (d, wd) in theweek + ) def formatweekday(self, day, width): """ @@ -388,10 +404,11 @@ def prmonth(self, theyear, themonth, w=0, l=0): """ print(self.formatmonth(theyear, themonth, w, l), end='') - def formatmonth(self, theyear, themonth, w=0, l=0): + def formatmonth(self, theyear, themonth, w=0, l=0, *, highlight_day=None): """ Return a month's calendar string (multi-line). """ + highlight_day = highlight_day.day if highlight_day else None w = max(2, w) l = max(1, l) s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1) @@ -400,11 +417,11 @@ def formatmonth(self, theyear, themonth, w=0, l=0): s += self.formatweekheader(w).rstrip() s += '\n' * l for week in self.monthdays2calendar(theyear, themonth): - s += self.formatweek(week, w).rstrip() + s += self.formatweek(week, w, highlight_day=highlight_day).rstrip() s += '\n' * l return s - def formatyear(self, theyear, w=2, l=1, c=6, m=3): + def formatyear(self, theyear, w=2, l=1, c=6, m=3, *, highlight_day=None): """ Returns a year's calendar as a multi-line string. """ @@ -428,15 +445,24 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3): headers = (header for k in months) a(formatstring(headers, colwidth, c).rstrip()) a('\n'*l) + + if highlight_day and highlight_day.month in months: + month_pos = months.index(highlight_day.month) + else: + month_pos = None + # max number of weeks for this row height = max(len(cal) for cal in row) for j in range(height): weeks = [] - for cal in row: + for k, cal in enumerate(row): if j >= len(cal): weeks.append('') else: - weeks.append(self.formatweek(cal[j], w)) + day = highlight_day.day if k == month_pos else None + weeks.append( + self.formatweek(cal[j], w, highlight_day=day) + ) a(formatstring(weeks, colwidth, c).rstrip()) a('\n' * l) return ''.join(v) @@ -765,6 +791,7 @@ def main(args=None): sys.exit(1) locale = options.locale, options.encoding + today = datetime.date.today() if options.type == "html": if options.month: @@ -781,7 +808,7 @@ def main(args=None): optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write if options.year is None: - write(cal.formatyearpage(datetime.date.today().year, **optdict)) + write(cal.formatyearpage(today.year, **optdict)) else: write(cal.formatyearpage(options.year, **optdict)) else: @@ -797,10 +824,15 @@ def main(args=None): if options.month is not None: _validate_month(options.month) if options.year is None: - result = cal.formatyear(datetime.date.today().year, **optdict) + optdict["highlight_day"] = today + result = cal.formatyear(today.year, **optdict) elif options.month is None: + if options.year == today.year: + optdict["highlight_day"] = today result = cal.formatyear(options.year, **optdict) else: + if options.year == today.year and options.month == today.month: + optdict["highlight_day"] = today result = cal.formatmonth(options.year, options.month, **optdict) write = sys.stdout.write if options.encoding: diff --git a/Misc/NEWS.d/next/Library/2024-12-29-00-33-34.gh-issue-128317.WgFina.rst b/Misc/NEWS.d/next/Library/2024-12-29-00-33-34.gh-issue-128317.WgFina.rst new file mode 100644 index 00000000000000..4441108014569e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-29-00-33-34.gh-issue-128317.WgFina.rst @@ -0,0 +1,2 @@ +Highlight today in colour in :mod:`calendar`'s CLI output. Patch by Hugo van +Kemenade. From b4f799b1e78ede17b41de9a2bc51b437a7e6dd74 Mon Sep 17 00:00:00 2001 From: Rian Hunter Date: Fri, 3 Jan 2025 05:07:07 -0800 Subject: [PATCH 349/427] gh-112015: Implement `ctypes.memoryview_at()` (GH-112018) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Serhiy Storchaka Co-authored-by: Petr Viktorin Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/ctypes.rst | 22 +++++++ Doc/whatsnew/3.14.rst | 8 +++ Lib/ctypes/__init__.py | 9 +++ Lib/test/test_ctypes/test_memfunctions.py | 60 ++++++++++++++++++- ...-11-12-21-53-40.gh-issue-112015.2WPRxE.rst | 5 ++ Modules/_ctypes/_ctypes.c | 17 ++++++ 6 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-12-21-53-40.gh-issue-112015.2WPRxE.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 09692e56d29a39..398cb92bac809a 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2182,6 +2182,28 @@ Utility functions .. audit-event:: ctypes.wstring_at ptr,size ctypes.wstring_at +.. function:: memoryview_at(ptr, size, readonly=False) + + Return a :class:`memoryview` object of length *size* that references memory + starting at *void \*ptr*. + + If *readonly* is true, the returned :class:`!memoryview` object can + not be used to modify the underlying memory. + (Changes made by other means will still be reflected in the returned + object.) + + This function is similar to :func:`string_at` with the key + difference of not making a copy of the specified memory. + It is a semantically equivalent (but more efficient) alternative to + ``memoryview((c_byte * size).from_address(ptr))``. + (While :meth:`~_CData.from_address` only takes integers, *ptr* can also + be given as a :class:`ctypes.POINTER` or a :func:`~ctypes.byref` object.) + + .. audit-event:: ctypes.memoryview_at address,size,readonly + + .. versionadded:: next + + .. _ctypes-data-types: Data types diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index cb9167300260cb..f365db37217e95 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -343,6 +343,14 @@ ctypes * On Windows, the :func:`~ctypes.CopyComPointer` function is now public. (Contributed by Jun Komoda in :gh:`127275`.) +* :func:`ctypes.memoryview_at` now exists to create a + :class:`memoryview` object that refers to the supplied pointer and + length. This works like :func:`ctypes.string_at` except it avoids a + buffer copy, and is typically useful when implementing pure Python + callback functions that are passed dynamically-sized buffers. + (Contributed by Rian Hunter in :gh:`112018`.) + + datetime -------- diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 2f2b0ca9f38633..8e2a2926f7a853 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -524,6 +524,7 @@ def WinError(code=None, descr=None): # functions from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memoryview_at_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -549,6 +550,14 @@ def string_at(ptr, size=-1): Return the byte string at void *ptr.""" return _string_at(ptr, size) +_memoryview_at = PYFUNCTYPE( + py_object, c_void_p, c_ssize_t, c_int)(_memoryview_at_addr) +def memoryview_at(ptr, size, readonly=False): + """memoryview_at(ptr, size[, readonly]) -> memoryview + + Return a memoryview representing the memory at void *ptr.""" + return _memoryview_at(ptr, size, bool(readonly)) + try: from _ctypes import _wstring_at_addr except ImportError: diff --git a/Lib/test/test_ctypes/test_memfunctions.py b/Lib/test/test_ctypes/test_memfunctions.py index 112b27ba48e07e..325487618137f6 100644 --- a/Lib/test/test_ctypes/test_memfunctions.py +++ b/Lib/test/test_ctypes/test_memfunctions.py @@ -5,7 +5,9 @@ create_string_buffer, string_at, create_unicode_buffer, wstring_at, memmove, memset, - c_char_p, c_byte, c_ubyte, c_wchar) + memoryview_at, c_void_p, + c_char_p, c_byte, c_ubyte, c_wchar, + addressof, byref) class MemFunctionsTest(unittest.TestCase): @@ -77,6 +79,62 @@ def test_wstring_at(self): self.assertEqual(wstring_at(a, 16), "Hello, World\0\0\0\0") self.assertEqual(wstring_at(a, 0), "") + def test_memoryview_at(self): + b = (c_byte * 10)() + + size = len(b) + for foreign_ptr in ( + b, + cast(b, c_void_p), + byref(b), + addressof(b), + ): + with self.subTest(foreign_ptr=type(foreign_ptr).__name__): + b[:] = b"initialval" + v = memoryview_at(foreign_ptr, size) + self.assertIsInstance(v, memoryview) + self.assertEqual(bytes(v), b"initialval") + + # test that writes to source buffer get reflected in memoryview + b[:] = b"0123456789" + self.assertEqual(bytes(v), b"0123456789") + + # test that writes to memoryview get reflected in source buffer + v[:] = b"9876543210" + self.assertEqual(bytes(b), b"9876543210") + + with self.assertRaises(ValueError): + memoryview_at(foreign_ptr, -1) + + with self.assertRaises(ValueError): + memoryview_at(foreign_ptr, sys.maxsize + 1) + + v0 = memoryview_at(foreign_ptr, 0) + self.assertEqual(bytes(v0), b'') + + def test_memoryview_at_readonly(self): + b = (c_byte * 10)() + + size = len(b) + for foreign_ptr in ( + b, + cast(b, c_void_p), + byref(b), + addressof(b), + ): + with self.subTest(foreign_ptr=type(foreign_ptr).__name__): + b[:] = b"initialval" + v = memoryview_at(foreign_ptr, size, readonly=True) + self.assertIsInstance(v, memoryview) + self.assertEqual(bytes(v), b"initialval") + + # test that writes to source buffer get reflected in memoryview + b[:] = b"0123456789" + self.assertEqual(bytes(v), b"0123456789") + + # test that writes to the memoryview are blocked + with self.assertRaises(TypeError): + v[:] = b"9876543210" if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-11-12-21-53-40.gh-issue-112015.2WPRxE.rst b/Misc/NEWS.d/next/Library/2023-11-12-21-53-40.gh-issue-112015.2WPRxE.rst new file mode 100644 index 00000000000000..4b58ec9d219eff --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-12-21-53-40.gh-issue-112015.2WPRxE.rst @@ -0,0 +1,5 @@ +:func:`ctypes.memoryview_at` now exists to create a +:class:`memoryview` object that refers to the supplied pointer and +length. This works like :func:`ctypes.string_at` except it avoids a +buffer copy, and is typically useful when implementing pure Python +callback functions that are passed dynamically-sized buffers. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index ac520ffaad6c90..ede95bdf98bf76 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5791,6 +5791,22 @@ wstring_at(const wchar_t *ptr, int size) return PyUnicode_FromWideChar(ptr, ssize); } +static PyObject * +memoryview_at(void *ptr, Py_ssize_t size, int readonly) +{ + if (PySys_Audit("ctypes.memoryview_at", "nni", + (Py_ssize_t)ptr, size, readonly) < 0) { + return NULL; + } + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "memoryview_at: size is negative (or overflowed): %zd", + size); + return NULL; + } + return PyMemoryView_FromMemory(ptr, size, + readonly ? PyBUF_READ : PyBUF_WRITE); +} static int _ctypes_add_types(PyObject *mod) @@ -5919,6 +5935,7 @@ _ctypes_add_objects(PyObject *mod) MOD_ADD("_string_at_addr", PyLong_FromVoidPtr(string_at)); MOD_ADD("_cast_addr", PyLong_FromVoidPtr(cast)); MOD_ADD("_wstring_at_addr", PyLong_FromVoidPtr(wstring_at)); + MOD_ADD("_memoryview_at_addr", PyLong_FromVoidPtr(memoryview_at)); /* If RTLD_LOCAL is not defined (Windows!), set it to zero. */ #if !HAVE_DECL_RTLD_LOCAL From 8522f8bacb4e2db4b901472c254efad6057e9fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:04:33 +0100 Subject: [PATCH 350/427] gh-111178: fix UBSan failures in `Modules/_csv.c` (GH-128243) Also: suppress unused return values --- Modules/_csv.c | 76 +++++++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/Modules/_csv.c b/Modules/_csv.c index 1a4dc3f1f55ace..7ca30e39e00c0c 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -77,7 +77,7 @@ _csv_traverse(PyObject *module, visitproc visit, void *arg) static void _csv_free(void *module) { - _csv_clear((PyObject *)module); + (void)_csv_clear((PyObject *)module); } typedef enum { @@ -151,6 +151,10 @@ typedef struct { PyObject *error_obj; /* cached error object */ } WriterObj; +#define _DialectObj_CAST(op) ((DialectObj *)(op)) +#define _ReaderObj_CAST(op) ((ReaderObj *)(op)) +#define _WriterObj_CAST(op) ((WriterObj *)(op)) + /* * DIALECT class */ @@ -176,32 +180,37 @@ get_char_or_None(Py_UCS4 c) } static PyObject * -Dialect_get_lineterminator(DialectObj *self, void *Py_UNUSED(ignored)) +Dialect_get_lineterminator(PyObject *op, void *Py_UNUSED(ignored)) { + DialectObj *self = _DialectObj_CAST(op); return Py_XNewRef(self->lineterminator); } static PyObject * -Dialect_get_delimiter(DialectObj *self, void *Py_UNUSED(ignored)) +Dialect_get_delimiter(PyObject *op, void *Py_UNUSED(ignored)) { + DialectObj *self = _DialectObj_CAST(op); return get_char_or_None(self->delimiter); } static PyObject * -Dialect_get_escapechar(DialectObj *self, void *Py_UNUSED(ignored)) +Dialect_get_escapechar(PyObject *op, void *Py_UNUSED(ignored)) { + DialectObj *self = _DialectObj_CAST(op); return get_char_or_None(self->escapechar); } static PyObject * -Dialect_get_quotechar(DialectObj *self, void *Py_UNUSED(ignored)) +Dialect_get_quotechar(PyObject *op, void *Py_UNUSED(ignored)) { + DialectObj *self = _DialectObj_CAST(op); return get_char_or_None(self->quotechar); } static PyObject * -Dialect_get_quoting(DialectObj *self, void *Py_UNUSED(ignored)) +Dialect_get_quoting(PyObject *op, void *Py_UNUSED(ignored)) { + DialectObj *self = _DialectObj_CAST(op); return PyLong_FromLong(self->quoting); } @@ -371,16 +380,16 @@ static struct PyMemberDef Dialect_memberlist[] = { #undef D_OFF static PyGetSetDef Dialect_getsetlist[] = { - { "delimiter", (getter)Dialect_get_delimiter}, - { "escapechar", (getter)Dialect_get_escapechar}, - { "lineterminator", (getter)Dialect_get_lineterminator}, - { "quotechar", (getter)Dialect_get_quotechar}, - { "quoting", (getter)Dialect_get_quoting}, + {"delimiter", Dialect_get_delimiter}, + {"escapechar", Dialect_get_escapechar}, + {"lineterminator", Dialect_get_lineterminator}, + {"quotechar", Dialect_get_quotechar}, + {"quoting", Dialect_get_quoting}, {NULL}, }; static void -Dialect_dealloc(DialectObj *self) +Dialect_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); @@ -594,15 +603,17 @@ PyDoc_STRVAR(Dialect_Type_doc, "The Dialect type records CSV parsing and generation options.\n"); static int -Dialect_clear(DialectObj *self) +Dialect_clear(PyObject *op) { + DialectObj *self = _DialectObj_CAST(op); Py_CLEAR(self->lineterminator); return 0; } static int -Dialect_traverse(DialectObj *self, visitproc visit, void *arg) +Dialect_traverse(PyObject *op, visitproc visit, void *arg) { + DialectObj *self = _DialectObj_CAST(op); Py_VISIT(self->lineterminator); Py_VISIT(Py_TYPE(self)); return 0; @@ -916,8 +927,10 @@ parse_reset(ReaderObj *self) } static PyObject * -Reader_iternext(ReaderObj *self) +Reader_iternext(PyObject *op) { + ReaderObj *self = _ReaderObj_CAST(op); + PyObject *fields = NULL; Py_UCS4 c; Py_ssize_t pos, linelen; @@ -982,11 +995,12 @@ Reader_iternext(ReaderObj *self) } static void -Reader_dealloc(ReaderObj *self) +Reader_dealloc(PyObject *op) { + ReaderObj *self = _ReaderObj_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - tp->tp_clear((PyObject *)self); + (void)tp->tp_clear(op); if (self->field != NULL) { PyMem_Free(self->field); self->field = NULL; @@ -996,8 +1010,9 @@ Reader_dealloc(ReaderObj *self) } static int -Reader_traverse(ReaderObj *self, visitproc visit, void *arg) +Reader_traverse(PyObject *op, visitproc visit, void *arg) { + ReaderObj *self = _ReaderObj_CAST(op); Py_VISIT(self->dialect); Py_VISIT(self->input_iter); Py_VISIT(self->fields); @@ -1006,8 +1021,9 @@ Reader_traverse(ReaderObj *self, visitproc visit, void *arg) } static int -Reader_clear(ReaderObj *self) +Reader_clear(PyObject *op) { + ReaderObj *self = _ReaderObj_CAST(op); Py_CLEAR(self->dialect); Py_CLEAR(self->input_iter); Py_CLEAR(self->fields); @@ -1303,8 +1319,9 @@ PyDoc_STRVAR(csv_writerow_doc, "elements will be converted to string."); static PyObject * -csv_writerow(WriterObj *self, PyObject *seq) +csv_writerow(PyObject *op, PyObject *seq) { + WriterObj *self = _WriterObj_CAST(op); DialectObj *dialect = self->dialect; PyObject *iter, *field, *line, *result; bool null_field = false; @@ -1412,7 +1429,7 @@ PyDoc_STRVAR(csv_writerows_doc, "elements will be converted to string."); static PyObject * -csv_writerows(WriterObj *self, PyObject *seqseq) +csv_writerows(PyObject *self, PyObject *seqseq) { PyObject *row_iter, *row_obj, *result; @@ -1437,9 +1454,9 @@ csv_writerows(WriterObj *self, PyObject *seqseq) } static struct PyMethodDef Writer_methods[] = { - { "writerow", (PyCFunction)csv_writerow, METH_O, csv_writerow_doc}, - { "writerows", (PyCFunction)csv_writerows, METH_O, csv_writerows_doc}, - { NULL, NULL } + {"writerow", csv_writerow, METH_O, csv_writerow_doc}, + {"writerows", csv_writerows, METH_O, csv_writerows_doc}, + {NULL, NULL, 0, NULL} /* sentinel */ }; #define W_OFF(x) offsetof(WriterObj, x) @@ -1452,8 +1469,9 @@ static struct PyMemberDef Writer_memberlist[] = { #undef W_OFF static int -Writer_traverse(WriterObj *self, visitproc visit, void *arg) +Writer_traverse(PyObject *op, visitproc visit, void *arg) { + WriterObj *self = _WriterObj_CAST(op); Py_VISIT(self->dialect); Py_VISIT(self->write); Py_VISIT(self->error_obj); @@ -1462,8 +1480,9 @@ Writer_traverse(WriterObj *self, visitproc visit, void *arg) } static int -Writer_clear(WriterObj *self) +Writer_clear(PyObject *op) { + WriterObj *self = _WriterObj_CAST(op); Py_CLEAR(self->dialect); Py_CLEAR(self->write); Py_CLEAR(self->error_obj); @@ -1471,11 +1490,12 @@ Writer_clear(WriterObj *self) } static void -Writer_dealloc(WriterObj *self) +Writer_dealloc(PyObject *op) { + WriterObj *self = _WriterObj_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - tp->tp_clear((PyObject *)self); + tp->tp_clear(op); if (self->rec != NULL) { PyMem_Free(self->rec); } From 621d4ff35ec099c1626f77955318c0bc1febc33d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:12:40 +0100 Subject: [PATCH 351/427] gh-111178: fix UBSan failures in `Modules/curses*.c` (GH-128244) * fix UBSan failures in `_cursesmodule.c` * fix UBSan failures in `_curses_panel.c` * suppress an unused return value --- Modules/_curses_panel.c | 7 +- Modules/_cursesmodule.c | 178 ++++++++++++++++++++++++---------------- 2 files changed, 114 insertions(+), 71 deletions(-) diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index bbbb62c9066df0..eecf7a1c8a1e56 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -62,7 +62,7 @@ _curses_panel_traverse(PyObject *mod, visitproc visit, void *arg) static void _curses_panel_free(void *mod) { - _curses_panel_clear((PyObject *) mod); + (void)_curses_panel_clear((PyObject *)mod); } /* Utility Functions */ @@ -101,6 +101,8 @@ typedef struct { PyCursesWindowObject *wo; /* for reference counts */ } PyCursesPanelObject; +#define _PyCursesPanelObject_CAST(op) ((PyCursesPanelObject *)(op)) + /* Some helper functions. The problem is that there's always a window associated with a panel. To ensure that Python's GC doesn't pull this window from under our feet we need to keep track of references @@ -277,9 +279,10 @@ PyCursesPanel_New(_curses_panel_state *state, PANEL *pan, } static void -PyCursesPanel_Dealloc(PyCursesPanelObject *po) +PyCursesPanel_Dealloc(PyObject *self) { PyObject *tp, *obj; + PyCursesPanelObject *po = _PyCursesPanelObject_CAST(self); tp = (PyObject *) Py_TYPE(po); obj = (PyObject *) panel_userptr(po->pan); diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 92961af381b9cb..c6835738348ff9 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -187,6 +187,8 @@ get_cursesmodule_state_by_win(PyCursesWindowObject *win) return get_cursesmodule_state_by_cls(Py_TYPE(win)); } +#define _PyCursesWindowObject_CAST(op) ((PyCursesWindowObject *)(op)) + /*[clinic input] module _curses class _curses.window "PyCursesWindowObject *" "clinic_state()->window_type" @@ -654,53 +656,80 @@ class component_converter(CConverter): PARSESTR - format string for argument parsing */ -#define Window_NoArgNoReturnFunction(X) \ - static PyObject *PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ - { return PyCursesCheckERR_ForWin(self, X(self->win), # X); } +#define Window_NoArgNoReturnFunction(X) \ + static PyObject *PyCursesWindow_ ## X \ + (PyObject *op, PyObject *Py_UNUSED(ignored)) \ + { \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + int code = X(self->win); \ + return PyCursesCheckERR_ForWin(self, code, # X); \ + } #define Window_NoArgTrueFalseFunction(X) \ static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ + (PyObject *op, PyObject *Py_UNUSED(ignored)) \ { \ - return PyBool_FromLong(X(self->win)); } + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + return PyBool_FromLong(X(self->win)); \ + } -#define Window_NoArgNoReturnVoidFunction(X) \ - static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ - { \ - X(self->win); Py_RETURN_NONE; } +#define Window_NoArgNoReturnVoidFunction(X) \ + static PyObject * PyCursesWindow_ ## X \ + (PyObject *op, PyObject *Py_UNUSED(ignored)) \ + { \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + X(self->win); \ + Py_RETURN_NONE; \ + } #define Window_NoArg2TupleReturnFunction(X, TYPE, ERGSTR) \ static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ + (PyObject *op, PyObject *Py_UNUSED(ignored)) \ { \ TYPE arg1, arg2; \ - X(self->win,arg1,arg2); return Py_BuildValue(ERGSTR, arg1, arg2); } + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + X(self->win, arg1, arg2); \ + return Py_BuildValue(ERGSTR, arg1, arg2); \ + } #define Window_OneArgNoReturnVoidFunction(X, TYPE, PARSESTR) \ static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *args) \ + (PyObject *op, PyObject *args) \ { \ TYPE arg1; \ - if (!PyArg_ParseTuple(args, PARSESTR, &arg1)) return NULL; \ - X(self->win,arg1); Py_RETURN_NONE; } + if (!PyArg_ParseTuple(args, PARSESTR, &arg1)) { \ + return NULL; \ + } \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + X(self->win, arg1); \ + Py_RETURN_NONE; \ + } #define Window_OneArgNoReturnFunction(X, TYPE, PARSESTR) \ static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *args) \ + (PyObject *op, PyObject *args) \ { \ TYPE arg1; \ - if (!PyArg_ParseTuple(args,PARSESTR, &arg1)) return NULL; \ - return PyCursesCheckERR_ForWin(self, X(self->win, arg1), # X); } + if (!PyArg_ParseTuple(args, PARSESTR, &arg1)) { \ + return NULL; \ + } \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + int code = X(self->win, arg1); \ + return PyCursesCheckERR_ForWin(self, code, # X); \ + } #define Window_TwoArgNoReturnFunction(X, TYPE, PARSESTR) \ static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *args) \ + (PyObject *op, PyObject *args) \ { \ TYPE arg1, arg2; \ - if (!PyArg_ParseTuple(args,PARSESTR, &arg1, &arg2)) return NULL; \ - return PyCursesCheckERR_ForWin(self, X(self->win, arg1, arg2), # X); } + if (!PyArg_ParseTuple(args,PARSESTR, &arg1, &arg2)) { \ + return NULL; \ + } \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + int code = X(self->win, arg1, arg2); \ + return PyCursesCheckERR_ForWin(self, code, # X); \ + } /* ------------- WINDOW routines --------------- */ @@ -1302,8 +1331,10 @@ the touchline() method so that the contents will be redisplayed by the next window refresh. [-clinic start generated code]*/ static PyObject * -PyCursesWindow_ChgAt(PyCursesWindowObject *self, PyObject *args) +PyCursesWindow_ChgAt(PyObject *op, PyObject *args) { + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int rtn; int x, y; int num = -1; @@ -1656,8 +1687,10 @@ Read a string from the user, with primitive line editing capacity. [-clinic start generated code]*/ static PyObject * -PyCursesWindow_GetStr(PyCursesWindowObject *self, PyObject *args) +PyCursesWindow_GetStr(PyObject *op, PyObject *args) { + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int x, y, n; char rtn[1024]; /* This should be big enough.. I hope */ int rtn2; @@ -1860,8 +1893,10 @@ from the characters. If n is specified, instr() returns a string at most n characters long (exclusive of the trailing NUL). [-clinic start generated code]*/ static PyObject * -PyCursesWindow_InStr(PyCursesWindowObject *self, PyObject *args) +PyCursesWindow_InStr(PyObject *op, PyObject *args) { + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int x, y, n; char rtn[1024]; /* This should be big enough.. I hope */ int rtn2; @@ -2557,14 +2592,17 @@ _curses_window_vline_impl(PyCursesWindowObject *self, int group_left_1, } static PyObject * -PyCursesWindow_get_encoding(PyCursesWindowObject *self, void *closure) +PyCursesWindow_get_encoding(PyObject *op, void *closure) { + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); return PyUnicode_FromString(self->encoding); } static int -PyCursesWindow_set_encoding(PyCursesWindowObject *self, PyObject *value, void *Py_UNUSED(ignored)) +PyCursesWindow_set_encoding(PyObject *op, PyObject *value, void *Py_UNUSED(ignored)) { + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + PyObject *ascii; char *encoding; @@ -2607,88 +2645,90 @@ static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_ATTRSET_METHODDEF _CURSES_WINDOW_BKGD_METHODDEF #ifdef HAVE_CURSES_WCHGAT - {"chgat", (PyCFunction)PyCursesWindow_ChgAt, METH_VARARGS}, + {"chgat", PyCursesWindow_ChgAt, METH_VARARGS}, #endif _CURSES_WINDOW_BKGDSET_METHODDEF _CURSES_WINDOW_BORDER_METHODDEF _CURSES_WINDOW_BOX_METHODDEF - {"clear", (PyCFunction)PyCursesWindow_wclear, METH_NOARGS}, - {"clearok", (PyCFunction)PyCursesWindow_clearok, METH_VARARGS}, - {"clrtobot", (PyCFunction)PyCursesWindow_wclrtobot, METH_NOARGS}, - {"clrtoeol", (PyCFunction)PyCursesWindow_wclrtoeol, METH_NOARGS}, - {"cursyncup", (PyCFunction)PyCursesWindow_wcursyncup, METH_NOARGS}, + {"clear", PyCursesWindow_wclear, METH_NOARGS}, + {"clearok", PyCursesWindow_clearok, METH_VARARGS}, + {"clrtobot", PyCursesWindow_wclrtobot, METH_NOARGS}, + {"clrtoeol", PyCursesWindow_wclrtoeol, METH_NOARGS}, + {"cursyncup", PyCursesWindow_wcursyncup, METH_NOARGS}, _CURSES_WINDOW_DELCH_METHODDEF - {"deleteln", (PyCFunction)PyCursesWindow_wdeleteln, METH_NOARGS}, + {"deleteln", PyCursesWindow_wdeleteln, METH_NOARGS}, _CURSES_WINDOW_DERWIN_METHODDEF _CURSES_WINDOW_ECHOCHAR_METHODDEF _CURSES_WINDOW_ENCLOSE_METHODDEF - {"erase", (PyCFunction)PyCursesWindow_werase, METH_NOARGS}, - {"getbegyx", (PyCFunction)PyCursesWindow_getbegyx, METH_NOARGS}, + {"erase", PyCursesWindow_werase, METH_NOARGS}, + {"getbegyx", PyCursesWindow_getbegyx, METH_NOARGS}, _CURSES_WINDOW_GETBKGD_METHODDEF _CURSES_WINDOW_GETCH_METHODDEF _CURSES_WINDOW_GETKEY_METHODDEF _CURSES_WINDOW_GET_WCH_METHODDEF - {"getmaxyx", (PyCFunction)PyCursesWindow_getmaxyx, METH_NOARGS}, - {"getparyx", (PyCFunction)PyCursesWindow_getparyx, METH_NOARGS}, - {"getstr", (PyCFunction)PyCursesWindow_GetStr, METH_VARARGS}, - {"getyx", (PyCFunction)PyCursesWindow_getyx, METH_NOARGS}, + {"getmaxyx", PyCursesWindow_getmaxyx, METH_NOARGS}, + {"getparyx", PyCursesWindow_getparyx, METH_NOARGS}, + {"getstr", PyCursesWindow_GetStr, METH_VARARGS}, + {"getyx", PyCursesWindow_getyx, METH_NOARGS}, _CURSES_WINDOW_HLINE_METHODDEF - {"idcok", (PyCFunction)PyCursesWindow_idcok, METH_VARARGS}, - {"idlok", (PyCFunction)PyCursesWindow_idlok, METH_VARARGS}, + {"idcok", PyCursesWindow_idcok, METH_VARARGS}, + {"idlok", PyCursesWindow_idlok, METH_VARARGS}, #ifdef HAVE_CURSES_IMMEDOK - {"immedok", (PyCFunction)PyCursesWindow_immedok, METH_VARARGS}, + {"immedok", PyCursesWindow_immedok, METH_VARARGS}, #endif _CURSES_WINDOW_INCH_METHODDEF _CURSES_WINDOW_INSCH_METHODDEF - {"insdelln", (PyCFunction)PyCursesWindow_winsdelln, METH_VARARGS}, - {"insertln", (PyCFunction)PyCursesWindow_winsertln, METH_NOARGS}, + {"insdelln", PyCursesWindow_winsdelln, METH_VARARGS}, + {"insertln", PyCursesWindow_winsertln, METH_NOARGS}, _CURSES_WINDOW_INSNSTR_METHODDEF _CURSES_WINDOW_INSSTR_METHODDEF - {"instr", (PyCFunction)PyCursesWindow_InStr, METH_VARARGS}, + {"instr", PyCursesWindow_InStr, METH_VARARGS}, _CURSES_WINDOW_IS_LINETOUCHED_METHODDEF - {"is_wintouched", (PyCFunction)PyCursesWindow_is_wintouched, METH_NOARGS}, - {"keypad", (PyCFunction)PyCursesWindow_keypad, METH_VARARGS}, - {"leaveok", (PyCFunction)PyCursesWindow_leaveok, METH_VARARGS}, - {"move", (PyCFunction)PyCursesWindow_wmove, METH_VARARGS}, - {"mvderwin", (PyCFunction)PyCursesWindow_mvderwin, METH_VARARGS}, - {"mvwin", (PyCFunction)PyCursesWindow_mvwin, METH_VARARGS}, - {"nodelay", (PyCFunction)PyCursesWindow_nodelay, METH_VARARGS}, - {"notimeout", (PyCFunction)PyCursesWindow_notimeout, METH_VARARGS}, + {"is_wintouched", PyCursesWindow_is_wintouched, METH_NOARGS}, + {"keypad", PyCursesWindow_keypad, METH_VARARGS}, + {"leaveok", PyCursesWindow_leaveok, METH_VARARGS}, + {"move", PyCursesWindow_wmove, METH_VARARGS}, + {"mvderwin", PyCursesWindow_mvderwin, METH_VARARGS}, + {"mvwin", PyCursesWindow_mvwin, METH_VARARGS}, + {"nodelay", PyCursesWindow_nodelay, METH_VARARGS}, + {"notimeout", PyCursesWindow_notimeout, METH_VARARGS}, _CURSES_WINDOW_NOUTREFRESH_METHODDEF _CURSES_WINDOW_OVERLAY_METHODDEF _CURSES_WINDOW_OVERWRITE_METHODDEF _CURSES_WINDOW_PUTWIN_METHODDEF _CURSES_WINDOW_REDRAWLN_METHODDEF - {"redrawwin", (PyCFunction)PyCursesWindow_redrawwin, METH_NOARGS}, + {"redrawwin", PyCursesWindow_redrawwin, METH_NOARGS}, _CURSES_WINDOW_REFRESH_METHODDEF #ifndef STRICT_SYSV_CURSES - {"resize", (PyCFunction)PyCursesWindow_wresize, METH_VARARGS}, + {"resize", PyCursesWindow_wresize, METH_VARARGS}, #endif _CURSES_WINDOW_SCROLL_METHODDEF - {"scrollok", (PyCFunction)PyCursesWindow_scrollok, METH_VARARGS}, + {"scrollok", PyCursesWindow_scrollok, METH_VARARGS}, _CURSES_WINDOW_SETSCRREG_METHODDEF - {"standend", (PyCFunction)PyCursesWindow_wstandend, METH_NOARGS}, - {"standout", (PyCFunction)PyCursesWindow_wstandout, METH_NOARGS}, + {"standend", PyCursesWindow_wstandend, METH_NOARGS}, + {"standout", PyCursesWindow_wstandout, METH_NOARGS}, {"subpad", (PyCFunction)_curses_window_subwin, METH_VARARGS, _curses_window_subwin__doc__}, _CURSES_WINDOW_SUBWIN_METHODDEF - {"syncdown", (PyCFunction)PyCursesWindow_wsyncdown, METH_NOARGS}, + {"syncdown", PyCursesWindow_wsyncdown, METH_NOARGS}, #ifdef HAVE_CURSES_SYNCOK - {"syncok", (PyCFunction)PyCursesWindow_syncok, METH_VARARGS}, + {"syncok", PyCursesWindow_syncok, METH_VARARGS}, #endif - {"syncup", (PyCFunction)PyCursesWindow_wsyncup, METH_NOARGS}, - {"timeout", (PyCFunction)PyCursesWindow_wtimeout, METH_VARARGS}, + {"syncup", PyCursesWindow_wsyncup, METH_NOARGS}, + {"timeout", PyCursesWindow_wtimeout, METH_VARARGS}, _CURSES_WINDOW_TOUCHLINE_METHODDEF - {"touchwin", (PyCFunction)PyCursesWindow_touchwin, METH_NOARGS}, - {"untouchwin", (PyCFunction)PyCursesWindow_untouchwin, METH_NOARGS}, + {"touchwin", PyCursesWindow_touchwin, METH_NOARGS}, + {"untouchwin", PyCursesWindow_untouchwin, METH_NOARGS}, _CURSES_WINDOW_VLINE_METHODDEF {NULL, NULL} /* sentinel */ }; static PyGetSetDef PyCursesWindow_getsets[] = { - {"encoding", - (getter)PyCursesWindow_get_encoding, - (setter)PyCursesWindow_set_encoding, - "the typecode character used to create the array"}, + { + "encoding", + PyCursesWindow_get_encoding, + PyCursesWindow_set_encoding, + "the typecode character used to create the array" + }, {NULL, NULL, NULL, NULL } /* sentinel */ }; From aad5ba4b6aafd3599d1737c3040b1719165e6e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:29:41 +0100 Subject: [PATCH 352/427] gh-111178: fix UBSan failures in `Objects/enumobject.c` (GH-128246) * fix UBSan failures for `enumobject` * fix UBSan failures for `reversedobject` --- Objects/enumobject.c | 57 +++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 556666779d8522..eb8952675269a2 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -23,6 +23,7 @@ typedef struct { PyObject* one; /* borrowed reference */ } enumobject; +#define _enumobject_CAST(op) ((enumobject *)(op)) /*[clinic input] @classmethod @@ -150,8 +151,9 @@ enumerate_vectorcall(PyObject *type, PyObject *const *args, } static void -enum_dealloc(enumobject *en) +enum_dealloc(PyObject *op) { + enumobject *en = _enumobject_CAST(op); PyObject_GC_UnTrack(en); Py_XDECREF(en->en_sit); Py_XDECREF(en->en_result); @@ -160,8 +162,9 @@ enum_dealloc(enumobject *en) } static int -enum_traverse(enumobject *en, visitproc visit, void *arg) +enum_traverse(PyObject *op, visitproc visit, void *arg) { + enumobject *en = _enumobject_CAST(op); Py_VISIT(en->en_sit); Py_VISIT(en->en_result); Py_VISIT(en->en_longindex); @@ -220,8 +223,9 @@ enum_next_long(enumobject *en, PyObject* next_item) } static PyObject * -enum_next(enumobject *en) +enum_next(PyObject *op) { + enumobject *en = _enumobject_CAST(op); PyObject *next_index; PyObject *next_item; PyObject *result = en->en_result; @@ -270,8 +274,9 @@ enum_next(enumobject *en) } static PyObject * -enum_reduce(enumobject *en, PyObject *Py_UNUSED(ignored)) +enum_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) { + enumobject *en = _enumobject_CAST(op); if (en->en_longindex != NULL) return Py_BuildValue("O(OO)", Py_TYPE(en), en->en_sit, en->en_longindex); else @@ -281,7 +286,7 @@ enum_reduce(enumobject *en, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef enum_methods[] = { - {"__reduce__", (PyCFunction)enum_reduce, METH_NOARGS, reduce_doc}, + {"__reduce__", enum_reduce, METH_NOARGS, reduce_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -293,7 +298,7 @@ PyTypeObject PyEnum_Type = { sizeof(enumobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)enum_dealloc, /* tp_dealloc */ + enum_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -311,12 +316,12 @@ PyTypeObject PyEnum_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ enum_new__doc__, /* tp_doc */ - (traverseproc)enum_traverse, /* tp_traverse */ + enum_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)enum_next, /* tp_iternext */ + enum_next, /* tp_iternext */ enum_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -329,7 +334,7 @@ PyTypeObject PyEnum_Type = { PyType_GenericAlloc, /* tp_alloc */ enum_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ - .tp_vectorcall = (vectorcallfunc)enumerate_vectorcall + .tp_vectorcall = enumerate_vectorcall }; /* Reversed Object ***************************************************************/ @@ -340,6 +345,8 @@ typedef struct { PyObject* seq; } reversedobject; +#define _reversedobject_CAST(op) ((reversedobject *)(op)) + /*[clinic input] @classmethod reversed.__new__ as reversed_new @@ -411,23 +418,26 @@ reversed_vectorcall(PyObject *type, PyObject * const*args, } static void -reversed_dealloc(reversedobject *ro) +reversed_dealloc(PyObject *op) { + reversedobject *ro = _reversedobject_CAST(op); PyObject_GC_UnTrack(ro); Py_XDECREF(ro->seq); Py_TYPE(ro)->tp_free(ro); } static int -reversed_traverse(reversedobject *ro, visitproc visit, void *arg) +reversed_traverse(PyObject *op, visitproc visit, void *arg) { + reversedobject *ro = _reversedobject_CAST(op); Py_VISIT(ro->seq); return 0; } static PyObject * -reversed_next(reversedobject *ro) +reversed_next(PyObject *op) { + reversedobject *ro = _reversedobject_CAST(op); PyObject *item; Py_ssize_t index = ro->index; @@ -447,8 +457,9 @@ reversed_next(reversedobject *ro) } static PyObject * -reversed_len(reversedobject *ro, PyObject *Py_UNUSED(ignored)) +reversed_len(PyObject *op, PyObject *Py_UNUSED(ignored)) { + reversedobject *ro = _reversedobject_CAST(op); Py_ssize_t position, seqsize; if (ro->seq == NULL) @@ -463,8 +474,9 @@ reversed_len(reversedobject *ro, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -reversed_reduce(reversedobject *ro, PyObject *Py_UNUSED(ignored)) +reversed_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) { + reversedobject *ro = _reversedobject_CAST(op); if (ro->seq) return Py_BuildValue("O(O)n", Py_TYPE(ro), ro->seq, ro->index); else @@ -472,8 +484,9 @@ reversed_reduce(reversedobject *ro, PyObject *Py_UNUSED(ignored)) } static PyObject * -reversed_setstate(reversedobject *ro, PyObject *state) +reversed_setstate(PyObject *op, PyObject *state) { + reversedobject *ro = _reversedobject_CAST(op); Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; @@ -493,9 +506,9 @@ reversed_setstate(reversedobject *ro, PyObject *state) PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); static PyMethodDef reversediter_methods[] = { - {"__length_hint__", (PyCFunction)reversed_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)reversed_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)reversed_setstate, METH_O, setstate_doc}, + {"__length_hint__", reversed_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", reversed_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", reversed_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -505,7 +518,7 @@ PyTypeObject PyReversed_Type = { sizeof(reversedobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)reversed_dealloc, /* tp_dealloc */ + reversed_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -523,12 +536,12 @@ PyTypeObject PyReversed_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ reversed_new__doc__, /* tp_doc */ - (traverseproc)reversed_traverse,/* tp_traverse */ + reversed_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)reversed_next, /* tp_iternext */ + reversed_next, /* tp_iternext */ reversediter_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -541,5 +554,5 @@ PyTypeObject PyReversed_Type = { PyType_GenericAlloc, /* tp_alloc */ reversed_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ - .tp_vectorcall = (vectorcallfunc)reversed_vectorcall, + .tp_vectorcall = reversed_vectorcall, }; From 56430320536a753236aadde59984c3c8f8777aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:35:05 +0100 Subject: [PATCH 353/427] gh-111178: fix UBSan failures in `Objects/tupleobject.c` (GH-128251) fix UBSan failures for `_PyTupleIterObject` --- Objects/tupleobject.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 49977726eadca9..002002eb455556 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -988,26 +988,29 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) /*********************** Tuple Iterator **************************/ +#define _PyTupleIterObject_CAST(op) ((_PyTupleIterObject *)(op)) static void -tupleiter_dealloc(_PyTupleIterObject *it) +tupleiter_dealloc(PyObject *self) { + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -tupleiter_traverse(_PyTupleIterObject *it, visitproc visit, void *arg) +tupleiter_traverse(PyObject *self, visitproc visit, void *arg) { + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); Py_VISIT(it->it_seq); return 0; } static PyObject * -tupleiter_next(PyObject *obj) +tupleiter_next(PyObject *self) { - _PyTupleIterObject *it = (_PyTupleIterObject *)obj; + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); PyTupleObject *seq; PyObject *item; @@ -1029,8 +1032,9 @@ tupleiter_next(PyObject *obj) } static PyObject * -tupleiter_len(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) +tupleiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); Py_ssize_t len = 0; if (it->it_seq) len = PyTuple_GET_SIZE(it->it_seq) - it->it_index; @@ -1040,13 +1044,14 @@ tupleiter_len(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -tupleiter_reduce(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) +tupleiter_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); /* _PyEval_GetBuiltin can invoke arbitrary code, * call must be before access of iterator pointers. * see issue #101765 */ + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); if (it->it_seq) return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); @@ -1055,8 +1060,9 @@ tupleiter_reduce(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) } static PyObject * -tupleiter_setstate(_PyTupleIterObject *it, PyObject *state) +tupleiter_setstate(PyObject *self, PyObject *state) { + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; @@ -1074,19 +1080,19 @@ PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); static PyMethodDef tupleiter_methods[] = { - {"__length_hint__", (PyCFunction)tupleiter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)tupleiter_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)tupleiter_setstate, METH_O, setstate_doc}, - {NULL, NULL} /* sentinel */ + {"__length_hint__", tupleiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", tupleiter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", tupleiter_setstate, METH_O, setstate_doc}, + {NULL, NULL, 0, NULL} /* sentinel */ }; PyTypeObject PyTupleIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "tuple_iterator", /* tp_name */ - sizeof(_PyTupleIterObject), /* tp_basicsize */ + sizeof(_PyTupleIterObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)tupleiter_dealloc, /* tp_dealloc */ + tupleiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1103,7 +1109,7 @@ PyTypeObject PyTupleIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)tupleiter_traverse, /* tp_traverse */ + tupleiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ From 1c9b0204796ddeaee710646871a4404b4cda1f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:36:41 +0100 Subject: [PATCH 354/427] gh-111178: fix UBSan failures in `Modules/zlibmodule.c` (GH-128252) --- Modules/zlibmodule.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 78dcce73cdaade..b90665ae7ef64a 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -221,6 +221,8 @@ typedef struct PyThread_type_lock lock; } compobject; +#define _compobject_CAST(op) ((compobject *)op) + static void zlib_error(zlibstate *state, z_stream zst, int err, const char *msg) { @@ -706,7 +708,7 @@ zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict) static void Dealloc(compobject *self) { - PyObject *type = (PyObject *)Py_TYPE(self); + PyTypeObject *type = Py_TYPE(self); PyThread_free_lock(self->lock); Py_XDECREF(self->unused_data); Py_XDECREF(self->unconsumed_tail); @@ -716,18 +718,20 @@ Dealloc(compobject *self) } static void -Comp_dealloc(compobject *self) +Comp_dealloc(PyObject *op) { + compobject *self = _compobject_CAST(op); if (self->is_initialised) - deflateEnd(&self->zst); + (void)deflateEnd(&self->zst); Dealloc(self); } static void -Decomp_dealloc(compobject *self) +Decomp_dealloc(PyObject *op) { + compobject *self = _compobject_CAST(op); if (self->is_initialised) - inflateEnd(&self->zst); + (void)inflateEnd(&self->zst); Dealloc(self); } From 4c14f03495724f2c52de2d34f1bfa35dd94757c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:51:22 +0100 Subject: [PATCH 355/427] gh-89083: improve UUIDv8 uniqueness tests (GH-128149) improve UUIDv8 uniqueness tests --- Lib/test/test_uuid.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 7bd26a8ca34b62..8f40dd97f42fdc 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -707,12 +707,16 @@ def test_uuid8(self): equal(u.int & 0x3fffffffffffffff, lo) def test_uuid8_uniqueness(self): - # Test that UUIDv8-generated values are unique - # (up to a negligible probability of failure). - u1 = self.uuid.uuid8() - u2 = self.uuid.uuid8() - self.assertNotEqual(u1.int, u2.int) - self.assertEqual(u1.version, u2.version) + # Test that UUIDv8-generated values are unique (up to a negligible + # probability of failure). There are 122 bits of entropy and assuming + # that the underlying mt-19937-based random generator is sufficiently + # good, it is unlikely to have a collision of two UUIDs. + N = 1000 + uuids = {self.uuid.uuid8() for _ in range(N)} + self.assertEqual(len(uuids), N) + + versions = {u.version for u in uuids} + self.assertSetEqual(versions, {8}) @support.requires_fork() def testIssue8621(self): From 46cb6340d7bad955edfc0a20f6a52dabc03b0932 Mon Sep 17 00:00:00 2001 From: Alexander Shadchin Date: Fri, 3 Jan 2025 21:47:58 +0300 Subject: [PATCH 356/427] gh-127903: Fix a crash on debug builds when calling `Objects/unicodeobject::_copy_characters`` (#127876) --- Lib/test/test_str.py | 7 +++++++ .../2024-12-13-14-17-24.gh-issue-127903.vemHSl.rst | 2 ++ Objects/unicodeobject.c | 9 ++++++--- 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-14-17-24.gh-issue-127903.vemHSl.rst diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index 4de6c1cba152bd..d1c9542c7d1317 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -7,6 +7,7 @@ """ import _string import codecs +import datetime import itertools import operator import pickle @@ -1908,6 +1909,12 @@ def test_utf8_decode_invalid_sequences(self): self.assertRaises(UnicodeDecodeError, (b'\xF4'+cb+b'\xBF\xBF').decode, 'utf-8') + def test_issue127903(self): + # gh-127903: ``_copy_characters`` crashes on DEBUG builds when + # there is nothing to copy. + d = datetime.datetime(2013, 11, 10, 14, 20, 59) + self.assertEqual(d.strftime('%z'), '') + def test_issue8271(self): # Issue #8271: during the decoding of an invalid UTF-8 byte sequence, # only the start byte and the continuation byte(s) are now considered diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-14-17-24.gh-issue-127903.vemHSl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-14-17-24.gh-issue-127903.vemHSl.rst new file mode 100644 index 00000000000000..ad479b52d1675c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-13-14-17-24.gh-issue-127903.vemHSl.rst @@ -0,0 +1,2 @@ +``Objects/unicodeobject.c``: fix a crash on DEBUG builds in ``_copy_characters`` +when there is nothing to copy. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 9f0a4d4785eda6..5e532ce0f348c4 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1463,11 +1463,14 @@ _copy_characters(PyObject *to, Py_ssize_t to_start, assert(PyUnicode_Check(from)); assert(from_start + how_many <= PyUnicode_GET_LENGTH(from)); - assert(PyUnicode_Check(to)); - assert(to_start + how_many <= PyUnicode_GET_LENGTH(to)); + assert(to == NULL || PyUnicode_Check(to)); - if (how_many == 0) + if (how_many == 0) { return 0; + } + + assert(to != NULL); + assert(to_start + how_many <= PyUnicode_GET_LENGTH(to)); from_kind = PyUnicode_KIND(from); from_data = PyUnicode_DATA(from); From befcfdfdaba15ecae38739ecabebd8046c1b1977 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 3 Jan 2025 14:14:57 -0500 Subject: [PATCH 357/427] gh-128400: Only show the current thread in `faulthandler` if the GIL is disabled (GH-128425) --- Doc/library/faulthandler.rst | 4 +++ Lib/test/test_faulthandler.py | 11 +++++-- ...-01-02-15-20-17.gh-issue-128400.UMiG4f.rst | 2 ++ Modules/faulthandler.c | 33 +++++++++++++++++-- 4 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-02-15-20-17.gh-issue-128400.UMiG4f.rst diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 4067d7912b88b2..b81da4af3cff58 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -91,6 +91,10 @@ Fault handler state The dump now mentions if a garbage collector collection is running if *all_threads* is true. + .. versionchanged:: next + Only the current thread is dumped if the :term:`GIL` is disabled to + prevent the risk of data races. + .. function:: disable() Disable the fault handler: uninstall the signal handlers installed by diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index fd56dee5d842ac..2088793cbb9387 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -100,7 +100,12 @@ def check_error(self, code, lineno, fatal_error, *, Raise an error if the output doesn't match the expected format. """ - if all_threads: + all_threads_disabled = ( + (not py_fatal_error) + and all_threads + and (not sys._is_gil_enabled()) + ) + if all_threads and not all_threads_disabled: if know_current_thread: header = 'Current thread 0x[0-9a-f]+' else: @@ -111,8 +116,10 @@ def check_error(self, code, lineno, fatal_error, *, if py_fatal_error: regex.append("Python runtime state: initialized") regex.append('') + if all_threads_disabled: + regex.append("") regex.append(fr'{header} \(most recent call first\):') - if garbage_collecting: + if garbage_collecting and not all_threads_disabled: regex.append(' Garbage-collecting') regex.append(fr' File "", line {lineno} in {function}') regex = '\n'.join(regex) diff --git a/Misc/NEWS.d/next/Library/2025-01-02-15-20-17.gh-issue-128400.UMiG4f.rst b/Misc/NEWS.d/next/Library/2025-01-02-15-20-17.gh-issue-128400.UMiG4f.rst new file mode 100644 index 00000000000000..f9d5f84224c8dc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-02-15-20-17.gh-issue-128400.UMiG4f.rst @@ -0,0 +1,2 @@ +Only show the current thread in :mod:`faulthandler` on the :term:`free +threaded ` build to prevent races. diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 2d16028a5232d0..b44b964b29484b 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_ceval.h" // _PyEval_IsGILEnabled #include "pycore_initconfig.h" // _PyStatus_ERR #include "pycore_pyerrors.h" // _Py_DumpExtensionModules #include "pycore_pystate.h" // _PyThreadState_GET() @@ -27,6 +28,8 @@ # include // getauxval() #endif +/* Sentinel to ignore all_threads on free-threading */ +#define FT_IGNORE_ALL_THREADS 2 /* Allocate at maximum 100 MiB of the stack to raise the stack overflow */ #define STACK_OVERFLOW_MAX_SIZE (100 * 1024 * 1024) @@ -201,10 +204,13 @@ faulthandler_dump_traceback(int fd, int all_threads, PyGILState_GetThisThreadState(). */ PyThreadState *tstate = PyGILState_GetThisThreadState(); - if (all_threads) { + if (all_threads == 1) { (void)_Py_DumpTracebackThreads(fd, NULL, tstate); } else { + if (all_threads == FT_IGNORE_ALL_THREADS) { + PUTS(fd, "\n"); + } if (tstate != NULL) _Py_DumpTraceback(fd, tstate); } @@ -271,6 +277,27 @@ faulthandler_disable_fatal_handler(fault_handler_t *handler) #endif } +static int +deduce_all_threads(void) +{ +#ifndef Py_GIL_DISABLED + return fatal_error.all_threads; +#else + if (fatal_error.all_threads == 0) { + return 0; + } + // We can't use _PyThreadState_GET, so use the stored GILstate one + PyThreadState *tstate = PyGILState_GetThisThreadState(); + if (tstate == NULL) { + return 0; + } + + /* In theory, it's safe to dump all threads if the GIL is enabled */ + return _PyEval_IsGILEnabled(tstate) + ? fatal_error.all_threads + : FT_IGNORE_ALL_THREADS; +#endif +} /* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals. @@ -325,7 +352,7 @@ faulthandler_fatal_error(int signum) PUTS(fd, "\n\n"); } - faulthandler_dump_traceback(fd, fatal_error.all_threads, + faulthandler_dump_traceback(fd, deduce_all_threads(), fatal_error.interp); _Py_DumpExtensionModules(fd, fatal_error.interp); @@ -401,7 +428,7 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) } } - faulthandler_dump_traceback(fd, fatal_error.all_threads, + faulthandler_dump_traceback(fd, deduce_all_threads(), fatal_error.interp); /* call the next exception handler */ From 4974bbdb290b61a2d8860de490fde1228c296753 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Fri, 3 Jan 2025 14:49:50 -0600 Subject: [PATCH 358/427] gh-128456: Use '-reorder-functions=cdsort' for BOLT builds (#128457) 'hfsort+' is deprecated in favor of 'cdsort'. --- configure | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 86b96d6208fe40..08fec03bef42dc 100755 --- a/configure +++ b/configure @@ -9403,7 +9403,7 @@ printf "%s\n" "$BOLT_INSTRUMENT_FLAGS" >&6; } printf %s "checking BOLT_APPLY_FLAGS... " >&6; } if test -z "${BOLT_APPLY_FLAGS}" then - BOLT_APPLY_FLAGS=" -update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot " + BOLT_APPLY_FLAGS=" -update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot " fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BOLT_APPLY_FLAGS" >&5 diff --git a/configure.ac b/configure.ac index 50b130f2c802b5..ecaf0fa9c43d49 100644 --- a/configure.ac +++ b/configure.ac @@ -2183,7 +2183,7 @@ then [m4_normalize(" -update-debug-sections -reorder-blocks=ext-tsp - -reorder-functions=hfsort+ + -reorder-functions=cdsort -split-functions -icf=1 -inline-all From b75ed951d4de8ba85349d80c8e7f097b3cd6052f Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Fri, 3 Jan 2025 14:51:06 -0600 Subject: [PATCH 359/427] gh-128354: Consistently use LIBS over LDFLAGS in library build checks (#128359) --- configure | 49 ++++++++++++++++++++++++------------------------- configure.ac | 35 +++++++++++++++++------------------ 2 files changed, 41 insertions(+), 43 deletions(-) diff --git a/configure b/configure index 08fec03bef42dc..aa88c74c61156c 100755 --- a/configure +++ b/configure @@ -14044,7 +14044,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBUUID_CFLAGS" - LDFLAGS="$LDFLAGS $LIBUUID_LIBS" + LIBS="$LIBS $LIBUUID_LIBS" for ac_header in uuid/uuid.h do : ac_fn_c_check_header_compile "$LINENO" "uuid/uuid.h" "ac_cv_header_uuid_uuid_h" "$ac_includes_default" @@ -14186,7 +14186,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBUUID_CFLAGS" - LDFLAGS="$LDFLAGS $LIBUUID_LIBS" + LIBS="$LIBS $LIBUUID_LIBS" for ac_header in uuid/uuid.h do : ac_fn_c_check_header_compile "$LINENO" "uuid/uuid.h" "ac_cv_header_uuid_uuid_h" "$ac_includes_default" @@ -15084,7 +15084,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" - LDFLAGS="$LDFLAGS $LIBFFI_LIBS" + LIBS="$LIBS $LIBFFI_LIBS" ac_fn_c_check_header_compile "$LINENO" "ffi.h" "ac_cv_header_ffi_h" "$ac_includes_default" if test "x$ac_cv_header_ffi_h" = xyes then : @@ -15166,7 +15166,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" - LDFLAGS="$LDFLAGS $LIBFFI_LIBS" + LIBS="$LIBS $LIBFFI_LIBS" ac_fn_c_check_header_compile "$LINENO" "ffi.h" "ac_cv_header_ffi_h" "$ac_includes_default" if test "x$ac_cv_header_ffi_h" = xyes then : @@ -15286,8 +15286,8 @@ save_LDFLAGS=$LDFLAGS save_LIBS=$LIBS - CFLAGS="$LIBFFI_CFLAGS $CFLAGS" - LDFLAGS="$LIBFFI_LIBS $LDFLAGS" + CFLAGS="$CFLAGS $LIBFFI_CFLAGS" + LIBS="$LIBS $LIBFFI_LIBS" @@ -15439,9 +15439,8 @@ save_LDFLAGS=$LDFLAGS save_LIBS=$LIBS - CPPFLAGS="$LIBFFI_CFLAGS $CPPFLAGS" - LDFLAGS="$LIBFFI_LIBS $LDFLAGS" - LIBS="$LIBFFI_LIBS $LIBS" + CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" + LIBS="$LIBS $LIBFFI_LIBS" if test "$cross_compiling" = yes then : ac_cv_ffi_complex_double_supported=no @@ -15614,8 +15613,8 @@ save_LDFLAGS=$LDFLAGS save_LIBS=$LIBS - CPPFLAGS="$LIBMPDEC_CFLAGS $CPPFLAGS" - LIBS="$LIBMPDEC_LIBS $LIBS" + CPPFLAGS="$CPPFLAGS $LIBMPDEC_CFLAGS" + LIBS="$LIBS $LIBMPDEC_LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -15885,7 +15884,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBSQLITE3_CFLAGS" - LIBS="$LIBSQLITE3_LIBS $LIBS" + LIBS="$LIBS $LIBSQLITE3_LIBS" ac_fn_c_check_header_compile "$LINENO" "sqlite3.h" "ac_cv_header_sqlite3_h" "$ac_includes_default" if test "x$ac_cv_header_sqlite3_h" = xyes @@ -16875,7 +16874,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $TCLTK_CFLAGS" - LIBS="$TCLTK_LIBS $LDFLAGS" + LIBS="$LIBS $TCLTK_LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -16942,7 +16941,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $GDBM_CFLAGS" - LDFLAGS="$GDBM_LIBS $LDFLAGS" + LIBS="$LIBS $GDBM_LIBS" for ac_header in gdbm.h do : ac_fn_c_check_header_compile "$LINENO" "gdbm.h" "ac_cv_header_gdbm_h" "$ac_includes_default" @@ -21492,7 +21491,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $ZLIB_CFLAGS" - LDFLAGS="$LDFLAGS $ZLIB_LIBS" + LIBS="$LIBS $ZLIB_LIBS" for ac_header in zlib.h do : ac_fn_c_check_header_compile "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default" @@ -21639,7 +21638,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $ZLIB_CFLAGS" - LDFLAGS="$LDFLAGS $ZLIB_LIBS" + LIBS="$LIBS $ZLIB_LIBS" for ac_header in zlib.h do : ac_fn_c_check_header_compile "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default" @@ -21876,7 +21875,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $BZIP2_CFLAGS" - LDFLAGS="$LDFLAGS $BZIP2_LIBS" + LIBS="$LIBS $BZIP2_LIBS" for ac_header in bzlib.h do : ac_fn_c_check_header_compile "$LINENO" "bzlib.h" "ac_cv_header_bzlib_h" "$ac_includes_default" @@ -21968,7 +21967,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $BZIP2_CFLAGS" - LDFLAGS="$LDFLAGS $BZIP2_LIBS" + LIBS="$LIBS $BZIP2_LIBS" for ac_header in bzlib.h do : ac_fn_c_check_header_compile "$LINENO" "bzlib.h" "ac_cv_header_bzlib_h" "$ac_includes_default" @@ -22124,7 +22123,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBLZMA_CFLAGS" - LDFLAGS="$LDFLAGS $LIBLZMA_LIBS" + LIBS="$LIBS $LIBLZMA_LIBS" for ac_header in lzma.h do : ac_fn_c_check_header_compile "$LINENO" "lzma.h" "ac_cv_header_lzma_h" "$ac_includes_default" @@ -22216,7 +22215,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBLZMA_CFLAGS" - LDFLAGS="$LDFLAGS $LIBLZMA_LIBS" + LIBS="$LIBS $LIBLZMA_LIBS" for ac_header in lzma.h do : ac_fn_c_check_header_compile "$LINENO" "lzma.h" "ac_cv_header_lzma_h" "$ac_includes_default" @@ -26370,7 +26369,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBREADLINE_CFLAGS" - LDFLAGS="$LDFLAGS $LIBREADLINE_LIBS" + LIBS="$LIBS $LIBREADLINE_LIBS" for ac_header in readline/readline.h do : ac_fn_c_check_header_compile "$LINENO" "readline/readline.h" "ac_cv_header_readline_readline_h" "$ac_includes_default" @@ -26459,7 +26458,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBREADLINE_CFLAGS" - LDFLAGS="$LDFLAGS $LIBREADLINE_LIBS" + LIBS="$LIBS $LIBREADLINE_LIBS" for ac_header in readline/readline.h do : ac_fn_c_check_header_compile "$LINENO" "readline/readline.h" "ac_cv_header_readline_readline_h" "$ac_includes_default" @@ -26621,7 +26620,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBEDIT_CFLAGS" - LDFLAGS="$LDFLAGS $LIBEDIT_LIBS" + LIBS="$LIBS $LIBEDIT_LIBS" for ac_header in editline/readline.h do : ac_fn_c_check_header_compile "$LINENO" "editline/readline.h" "ac_cv_header_editline_readline_h" "$ac_includes_default" @@ -26712,7 +26711,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBEDIT_CFLAGS" - LDFLAGS="$LDFLAGS $LIBEDIT_LIBS" + LIBS="$LIBS $LIBEDIT_LIBS" for ac_header in editline/readline.h do : ac_fn_c_check_header_compile "$LINENO" "editline/readline.h" "ac_cv_header_editline_readline_h" "$ac_includes_default" @@ -26830,7 +26829,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $READLINE_CFLAGS" - LIBS="$READLINE_LIBS $LIBS" + LIBS="$LIBS $READLINE_LIBS" LIBS_SAVE=$LIBS diff --git a/configure.ac b/configure.ac index ecaf0fa9c43d49..9e131ed1a2dc98 100644 --- a/configure.ac +++ b/configure.ac @@ -3701,7 +3701,7 @@ AS_VAR_IF([have_uuid], [missing], [ ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBUUID_CFLAGS" - LDFLAGS="$LDFLAGS $LIBUUID_LIBS" + LIBS="$LIBS $LIBUUID_LIBS" AC_CHECK_HEADERS([uuid/uuid.h], [ PY_CHECK_LIB([uuid], [uuid_generate_time], [have_uuid=yes]) PY_CHECK_LIB([uuid], [uuid_generate_time_safe], @@ -3971,7 +3971,7 @@ AS_VAR_IF([have_libffi], [missing], [ PKG_CHECK_MODULES([LIBFFI], [libffi], [have_libffi=yes], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" - LDFLAGS="$LDFLAGS $LIBFFI_LIBS" + LIBS="$LIBS $LIBFFI_LIBS" AC_CHECK_HEADER([ffi.h], [ AC_CHECK_LIB([ffi], [ffi_call], [ have_libffi=yes @@ -4005,8 +4005,8 @@ AS_VAR_IF([have_libffi], [yes], [ AS_VAR_IF([ac_cv_lib_dl_dlopen], [yes], [AS_VAR_APPEND([LIBFFI_LIBS], [" -ldl"])]) WITH_SAVE_ENV([ - CFLAGS="$LIBFFI_CFLAGS $CFLAGS" - LDFLAGS="$LIBFFI_LIBS $LDFLAGS" + CFLAGS="$CFLAGS $LIBFFI_CFLAGS" + LIBS="$LIBS $LIBFFI_LIBS" PY_CHECK_FUNC([ffi_prep_cif_var], [@%:@include ]) PY_CHECK_FUNC([ffi_prep_closure_loc], [@%:@include ]) @@ -4021,9 +4021,8 @@ AS_VAR_IF([have_libffi], [yes], [ # AC_CACHE_CHECK([libffi has complex type support], [ac_cv_ffi_complex_double_supported], [WITH_SAVE_ENV([ - CPPFLAGS="$LIBFFI_CFLAGS $CPPFLAGS" - LDFLAGS="$LIBFFI_LIBS $LDFLAGS" - LIBS="$LIBFFI_LIBS $LIBS" + CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" + LIBS="$LIBS $LIBFFI_LIBS" AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include @@ -4086,8 +4085,8 @@ AS_VAR_IF( AS_VAR_IF([with_system_libmpdec], [yes], [WITH_SAVE_ENV([ - CPPFLAGS="$LIBMPDEC_CFLAGS $CPPFLAGS" - LIBS="$LIBMPDEC_LIBS $LIBS" + CPPFLAGS="$CPPFLAGS $LIBMPDEC_CFLAGS" + LIBS="$LIBS $LIBMPDEC_LIBS" AC_LINK_IFELSE([ AC_LANG_PROGRAM([ @@ -4220,7 +4219,7 @@ WITH_SAVE_ENV([ dnl bpo-45774/GH-29507: The CPP check in AC_CHECK_HEADER can fail on FreeBSD, dnl hence CPPFLAGS instead of CFLAGS. CPPFLAGS="$CPPFLAGS $LIBSQLITE3_CFLAGS" - LIBS="$LIBSQLITE3_LIBS $LIBS" + LIBS="$LIBS $LIBSQLITE3_LIBS" AC_CHECK_HEADER([sqlite3.h], [ have_sqlite3=yes @@ -4323,7 +4322,7 @@ AS_CASE([$ac_sys_system], WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $TCLTK_CFLAGS" - LIBS="$TCLTK_LIBS $LDFLAGS" + LIBS="$LIBS $TCLTK_LIBS" AC_LINK_IFELSE([ AC_LANG_PROGRAM([ @@ -4365,7 +4364,7 @@ AC_ARG_VAR([GDBM_CFLAGS], [C compiler flags for gdbm]) AC_ARG_VAR([GDBM_LIBS], [additional linker flags for gdbm]) WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $GDBM_CFLAGS" - LDFLAGS="$GDBM_LIBS $LDFLAGS" + LIBS="$LIBS $GDBM_LIBS" AC_CHECK_HEADERS([gdbm.h], [ AC_CHECK_LIB([gdbm], [gdbm_open], [ have_gdbm=yes @@ -5310,7 +5309,7 @@ PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.0], [ ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $ZLIB_CFLAGS" - LDFLAGS="$LDFLAGS $ZLIB_LIBS" + LIBS="$LIBS $ZLIB_LIBS" AC_CHECK_HEADERS([zlib.h], [ PY_CHECK_LIB([z], [gzread], [have_zlib=yes], [have_zlib=no]) ], [have_zlib=no]) @@ -5334,7 +5333,7 @@ PY_CHECK_EMSCRIPTEN_PORT([BZIP2], [-sUSE_BZIP2]) PKG_CHECK_MODULES([BZIP2], [bzip2], [have_bzip2=yes], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $BZIP2_CFLAGS" - LDFLAGS="$LDFLAGS $BZIP2_LIBS" + LIBS="$LIBS $BZIP2_LIBS" AC_CHECK_HEADERS([bzlib.h], [ AC_CHECK_LIB([bz2], [BZ2_bzCompress], [have_bzip2=yes], [have_bzip2=no]) ], [have_bzip2=no]) @@ -5348,7 +5347,7 @@ PKG_CHECK_MODULES([BZIP2], [bzip2], [have_bzip2=yes], [ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [have_liblzma=yes], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBLZMA_CFLAGS" - LDFLAGS="$LDFLAGS $LIBLZMA_LIBS" + LIBS="$LIBS $LIBLZMA_LIBS" AC_CHECK_HEADERS([lzma.h], [ AC_CHECK_LIB([lzma], [lzma_easy_encoder], [have_liblzma=yes], [have_liblzma=no]) ], [have_liblzma=no]) @@ -6342,7 +6341,7 @@ AS_VAR_IF([with_readline], [readline], [ ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBREADLINE_CFLAGS" - LDFLAGS="$LDFLAGS $LIBREADLINE_LIBS" + LIBS="$LIBS $LIBREADLINE_LIBS" AC_CHECK_HEADERS([readline/readline.h], [ AC_CHECK_LIB([readline], [readline], [ LIBREADLINE=readline @@ -6363,7 +6362,7 @@ AS_VAR_IF([with_readline], [edit], [ ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBEDIT_CFLAGS" - LDFLAGS="$LDFLAGS $LIBEDIT_LIBS" + LIBS="$LIBS $LIBEDIT_LIBS" AC_CHECK_HEADERS([editline/readline.h], [ AC_CHECK_LIB([edit], [readline], [ LIBREADLINE=edit @@ -6387,7 +6386,7 @@ AS_VAR_IF([with_readline], [no], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $READLINE_CFLAGS" - LIBS="$READLINE_LIBS $LIBS" + LIBS="$LIBS $READLINE_LIBS" LIBS_SAVE=$LIBS m4_define([readline_includes], [ From f1574859d7d6cd259f867194762f04b72ef2c340 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 3 Jan 2025 16:48:47 -0500 Subject: [PATCH 360/427] gh-125985: Fix `cmodule_function()` scaling benchmark (#128460) Add a separate benchmark that measures the effect of `_PyObject_LookupSpecial()` on scaling. In the process of cleaning up the scaling benchmarks for inclusion, I unintentionally changed the "cmodule_function" benchmark to pass an `int` to `math.floor()` instead of a `float`, which causes it to use the `_PyObject_LookupSpecial()` code path. `_PyObject_LookupSpecial()` has its own scaling issues that we want to measure separately from calling a function on a C module. --- Tools/ftscalingbench/ftscalingbench.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 767aeae9349070..364c465bc91b0b 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -54,8 +54,16 @@ def object_cfunction(): @register_benchmark def cmodule_function(): - for i in range(1000 * WORK_SCALE): - math.floor(i * i) + N = 1000 * WORK_SCALE + for i in range(N): + math.cos(i / N) + +@register_benchmark +def object_lookup_special(): + # round() uses `_PyObject_LookupSpecial()` internally. + N = 1000 * WORK_SCALE + for i in range(N): + round(i / N) @register_benchmark def mult_constant(): @@ -206,7 +214,7 @@ def benchmark(func): color = "\x1b[33m" # yellow reset_color = "\x1b[0m" - print(f"{color}{func.__name__:<18} {round(factor, 1):>4}x {direction}{reset_color}") + print(f"{color}{func.__name__:<25} {round(factor, 1):>4}x {direction}{reset_color}") def determine_num_threads_and_affinity(): if sys.platform != "linux": From 513a4efa75bf78c9d629ddabc9516fb058787289 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 4 Jan 2025 14:18:22 +0530 Subject: [PATCH 361/427] gh-128002: fix many thread safety issues in asyncio (#128147) * Makes `_asyncio.Task` and `_asyncio.Future` thread-safe by adding critical sections * Add assertions to check for thread safety checking locking of object by critical sections in internal functions * Make `_asyncio.all_tasks` thread safe when eager tasks are used * Add a thread safety test --- Lib/test/test_asyncio/test_free_threading.py | 82 +++ Modules/_asynciomodule.c | 494 +++++++++++------ Modules/clinic/_asynciomodule.c.h | 550 ++++++++++++++++++- 3 files changed, 951 insertions(+), 175 deletions(-) create mode 100644 Lib/test/test_asyncio/test_free_threading.py diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py new file mode 100644 index 00000000000000..90bddbf3a9dda1 --- /dev/null +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -0,0 +1,82 @@ +import asyncio +import unittest +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + +def tearDownModule(): + asyncio._set_event_loop_policy(None) + + +class TestFreeThreading: + def test_all_tasks_race(self) -> None: + async def main(): + loop = asyncio.get_running_loop() + future = loop.create_future() + + async def coro(): + await future + + tasks = set() + + async with asyncio.TaskGroup() as tg: + for _ in range(100): + tasks.add(tg.create_task(coro())) + + all_tasks = self.all_tasks(loop) + self.assertEqual(len(all_tasks), 101) + + for task in all_tasks: + self.assertEqual(task.get_loop(), loop) + self.assertFalse(task.done()) + + current = self.current_task() + self.assertEqual(current.get_loop(), loop) + self.assertSetEqual(all_tasks, tasks | {current}) + future.set_result(None) + + def runner(): + with asyncio.Runner() as runner: + loop = runner.get_loop() + loop.set_task_factory(self.factory) + runner.run(main()) + + threads = [] + + for _ in range(10): + thread = Thread(target=runner) + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + +class TestPyFreeThreading(TestFreeThreading, TestCase): + all_tasks = staticmethod(asyncio.tasks._py_all_tasks) + current_task = staticmethod(asyncio.tasks._py_current_task) + + def factory(self, loop, coro, context=None): + return asyncio.tasks._PyTask(coro, loop=loop, context=context) + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestCFreeThreading(TestFreeThreading, TestCase): + all_tasks = staticmethod(getattr(asyncio.tasks, "_c_all_tasks", None)) + current_task = staticmethod(getattr(asyncio.tasks, "_c_current_task", None)) + + def factory(self, loop, coro, context=None): + return asyncio.tasks._CTask(coro, loop=loop, context=context) + + +class TestEagerPyFreeThreading(TestPyFreeThreading): + def factory(self, loop, coro, context=None): + return asyncio.tasks._PyTask(coro, loop=loop, context=context, eager_start=True) + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestEagerCFreeThreading(TestCFreeThreading, TestCase): + def factory(self, loop, coro, context=None): + return asyncio.tasks._CTask(coro, loop=loop, context=context, eager_start=True) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 74db4c74af905a..b8b184af04a7cb 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -364,6 +364,8 @@ future_ensure_alive(FutureObj *fut) static int future_schedule_callbacks(asyncio_state *state, FutureObj *fut) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (fut->fut_callback0 != NULL) { /* There's a 1st callback */ @@ -481,6 +483,8 @@ future_init(FutureObj *fut, PyObject *loop) static PyObject * future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (future_ensure_alive(fut)) { return NULL; } @@ -503,6 +507,8 @@ future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) static PyObject * future_set_exception(asyncio_state *state, FutureObj *fut, PyObject *exc) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + PyObject *exc_val = NULL; if (fut->fut_state != STATE_PENDING) { @@ -569,6 +575,8 @@ future_set_exception(asyncio_state *state, FutureObj *fut, PyObject *exc) static PyObject * create_cancelled_error(asyncio_state *state, FutureObj *fut) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + PyObject *exc; if (fut->fut_cancelled_exc != NULL) { /* transfer ownership */ @@ -588,6 +596,8 @@ create_cancelled_error(asyncio_state *state, FutureObj *fut) static void future_set_cancelled_error(asyncio_state *state, FutureObj *fut) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + PyObject *exc = create_cancelled_error(state, fut); if (exc == NULL) { return; @@ -599,6 +609,8 @@ future_set_cancelled_error(asyncio_state *state, FutureObj *fut) static int future_get_result(asyncio_state *state, FutureObj *fut, PyObject **result) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (fut->fut_state == STATE_CANCELLED) { future_set_cancelled_error(state, fut); return -1; @@ -632,6 +644,8 @@ static PyObject * future_add_done_callback(asyncio_state *state, FutureObj *fut, PyObject *arg, PyObject *ctx) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (!future_is_alive(fut)) { PyErr_SetString(PyExc_RuntimeError, "uninitialized Future object"); return NULL; @@ -706,6 +720,8 @@ future_add_done_callback(asyncio_state *state, FutureObj *fut, PyObject *arg, static PyObject * future_cancel(asyncio_state *state, FutureObj *fut, PyObject *msg) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + fut->fut_log_tb = 0; if (fut->fut_state != STATE_PENDING) { @@ -787,6 +803,7 @@ FutureObj_traverse(FutureObj *fut, visitproc visit, void *arg) } /*[clinic input] +@critical_section _asyncio.Future.result Return the result this future represents. @@ -798,7 +815,7 @@ the future is done and has an exception set, this exception is raised. static PyObject * _asyncio_Future_result_impl(FutureObj *self) -/*[clinic end generated code: output=f35f940936a4b1e5 input=49ecf9cf5ec50dc5]*/ +/*[clinic end generated code: output=f35f940936a4b1e5 input=61d89f48e4c8b670]*/ { asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *result; @@ -827,6 +844,7 @@ _asyncio_Future_result_impl(FutureObj *self) } /*[clinic input] +@critical_section _asyncio.Future.exception cls: defining_class @@ -842,7 +860,7 @@ InvalidStateError. static PyObject * _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=ce75576b187c905b input=3faf15c22acdb60d]*/ +/*[clinic end generated code: output=ce75576b187c905b input=647d1fd1fc403301]*/ { if (!future_is_alive(self)) { asyncio_state *state = get_asyncio_state_by_cls(cls); @@ -873,6 +891,7 @@ _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _asyncio.Future.set_result cls: defining_class @@ -888,7 +907,7 @@ InvalidStateError. static PyObject * _asyncio_Future_set_result_impl(FutureObj *self, PyTypeObject *cls, PyObject *result) -/*[clinic end generated code: output=99afbbe78f99c32d input=d5a41c1e353acc2e]*/ +/*[clinic end generated code: output=99afbbe78f99c32d input=4069306f03a3b6ee]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -896,6 +915,7 @@ _asyncio_Future_set_result_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.set_exception cls: defining_class @@ -911,7 +931,7 @@ InvalidStateError. static PyObject * _asyncio_Future_set_exception_impl(FutureObj *self, PyTypeObject *cls, PyObject *exception) -/*[clinic end generated code: output=0a5e8b5a52f058d6 input=a245cd49d3df939b]*/ +/*[clinic end generated code: output=0a5e8b5a52f058d6 input=b6eab43a389bc966]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -919,6 +939,7 @@ _asyncio_Future_set_exception_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.add_done_callback cls: defining_class @@ -937,7 +958,7 @@ scheduled with call_soon. static PyObject * _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn, PyObject *context) -/*[clinic end generated code: output=922e9a4cbd601167 input=599261c521458cc2]*/ +/*[clinic end generated code: output=922e9a4cbd601167 input=37d97f941beb7b3e]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); if (context == NULL) { @@ -953,6 +974,7 @@ _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.remove_done_callback cls: defining_class @@ -967,7 +989,7 @@ Returns the number of callbacks removed. static PyObject * _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn) -/*[clinic end generated code: output=2da35ccabfe41b98 input=c7518709b86fc747]*/ +/*[clinic end generated code: output=2da35ccabfe41b98 input=3afbc9f6a673091b]*/ { PyObject *newlist; Py_ssize_t len, i, j=0; @@ -1076,6 +1098,7 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.cancel cls: defining_class @@ -1092,7 +1115,7 @@ return True. static PyObject * _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, PyObject *msg) -/*[clinic end generated code: output=074956f35904b034 input=bba8f8b786941a94]*/ +/*[clinic end generated code: output=074956f35904b034 input=44ab4003da839970]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -1100,6 +1123,7 @@ _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.cancelled Return True if the future was cancelled. @@ -1107,7 +1131,7 @@ Return True if the future was cancelled. static PyObject * _asyncio_Future_cancelled_impl(FutureObj *self) -/*[clinic end generated code: output=145197ced586357d input=943ab8b7b7b17e45]*/ +/*[clinic end generated code: output=145197ced586357d input=9b8644819a675416]*/ { if (future_is_alive(self) && self->fut_state == STATE_CANCELLED) { Py_RETURN_TRUE; @@ -1118,6 +1142,7 @@ _asyncio_Future_cancelled_impl(FutureObj *self) } /*[clinic input] +@critical_section _asyncio.Future.done Return True if the future is done. @@ -1128,7 +1153,7 @@ future was cancelled. static PyObject * _asyncio_Future_done_impl(FutureObj *self) -/*[clinic end generated code: output=244c5ac351145096 input=28d7b23fdb65d2ac]*/ +/*[clinic end generated code: output=244c5ac351145096 input=7204d3cc63bef7f3]*/ { if (!future_is_alive(self) || self->fut_state == STATE_PENDING) { Py_RETURN_FALSE; @@ -1139,6 +1164,7 @@ _asyncio_Future_done_impl(FutureObj *self) } /*[clinic input] +@critical_section _asyncio.Future.get_loop cls: defining_class @@ -1149,17 +1175,24 @@ Return the event loop the Future is bound to. static PyObject * _asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=f50ea6c374d9ee97 input=163c2c498b45a1f0]*/ +/*[clinic end generated code: output=f50ea6c374d9ee97 input=f3ce629bfd9f45c1]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) return Py_NewRef(self->fut_loop); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._asyncio_future_blocking +[clinic start generated code]*/ + static PyObject * -FutureObj_get_blocking(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__asyncio_future_blocking_get_impl(FutureObj *self) +/*[clinic end generated code: output=a558a2c51e38823b input=58da92efc03b617d]*/ { - if (future_is_alive(fut) && fut->fut_blocking) { + if (future_is_alive(self) && self->fut_blocking) { Py_RETURN_TRUE; } else { @@ -1167,31 +1200,47 @@ FutureObj_get_blocking(FutureObj *fut, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@setter +_asyncio.Future._asyncio_future_blocking +[clinic start generated code]*/ + static int -FutureObj_set_blocking(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignored)) +_asyncio_Future__asyncio_future_blocking_set_impl(FutureObj *self, + PyObject *value) +/*[clinic end generated code: output=0686d1cb024a7453 input=3fd4a5f95df788b7]*/ + { - if (future_ensure_alive(fut)) { + if (future_ensure_alive(self)) { return -1; } - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - int is_true = PyObject_IsTrue(val); + int is_true = PyObject_IsTrue(value); if (is_true < 0) { return -1; } - fut->fut_blocking = is_true; + self->fut_blocking = is_true; return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._log_traceback +[clinic start generated code]*/ + static PyObject * -FutureObj_get_log_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__log_traceback_get_impl(FutureObj *self) +/*[clinic end generated code: output=2724433b238593c7 input=91e5144ea4117d8e]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) - if (fut->fut_log_tb) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) + if (self->fut_log_tb) { Py_RETURN_TRUE; } else { @@ -1199,14 +1248,21 @@ FutureObj_get_log_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@setter +_asyncio.Future._log_traceback +[clinic start generated code]*/ + static int -FutureObj_set_log_traceback(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignored)) +_asyncio_Future__log_traceback_set_impl(FutureObj *self, PyObject *value) +/*[clinic end generated code: output=9ce8e19504f42f54 input=30ac8217754b08c2]*/ { - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - int is_true = PyObject_IsTrue(val); + int is_true = PyObject_IsTrue(value); if (is_true < 0) { return -1; } @@ -1215,31 +1271,44 @@ FutureObj_set_log_traceback(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignor "_log_traceback can only be set to False"); return -1; } - fut->fut_log_tb = is_true; + self->fut_log_tb = is_true; return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._loop +[clinic start generated code]*/ static PyObject * -FutureObj_get_loop(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__loop_get_impl(FutureObj *self) +/*[clinic end generated code: output=5ba31563eecfeedf input=0337130bc5781670]*/ { - if (!future_is_alive(fut)) { + if (!future_is_alive(self)) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_loop); + return Py_NewRef(self->fut_loop); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._callbacks +[clinic start generated code]*/ + static PyObject * -FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__callbacks_get_impl(FutureObj *self) +/*[clinic end generated code: output=b40d360505fcc583 input=7a466649530c01bb]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) Py_ssize_t len = 0; - if (fut->fut_callback0 != NULL) { + if (self->fut_callback0 != NULL) { len++; } - if (fut->fut_callbacks != NULL) { - len += PyList_GET_SIZE(fut->fut_callbacks); + if (self->fut_callbacks != NULL) { + len += PyList_GET_SIZE(self->fut_callbacks); } if (len == 0) { @@ -1252,22 +1321,22 @@ FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored)) } Py_ssize_t i = 0; - if (fut->fut_callback0 != NULL) { + if (self->fut_callback0 != NULL) { PyObject *tup0 = PyTuple_New(2); if (tup0 == NULL) { Py_DECREF(callbacks); return NULL; } - PyTuple_SET_ITEM(tup0, 0, Py_NewRef(fut->fut_callback0)); - assert(fut->fut_context0 != NULL); - PyTuple_SET_ITEM(tup0, 1, Py_NewRef(fut->fut_context0)); + PyTuple_SET_ITEM(tup0, 0, Py_NewRef(self->fut_callback0)); + assert(self->fut_context0 != NULL); + PyTuple_SET_ITEM(tup0, 1, Py_NewRef(self->fut_context0)); PyList_SET_ITEM(callbacks, i, tup0); i++; } - if (fut->fut_callbacks != NULL) { - for (Py_ssize_t j = 0; j < PyList_GET_SIZE(fut->fut_callbacks); j++) { - PyObject *cb = PyList_GET_ITEM(fut->fut_callbacks, j); + if (self->fut_callbacks != NULL) { + for (Py_ssize_t j = 0; j < PyList_GET_SIZE(self->fut_callbacks); j++) { + PyObject *cb = PyList_GET_ITEM(self->fut_callbacks, j); Py_INCREF(cb); PyList_SET_ITEM(callbacks, i, cb); i++; @@ -1277,68 +1346,110 @@ FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored)) return callbacks; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._result +[clinic start generated code]*/ + static PyObject * -FutureObj_get_result(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__result_get_impl(FutureObj *self) +/*[clinic end generated code: output=6877e8ce97333873 input=624f8e28e67f2636]*/ + { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) - if (fut->fut_result == NULL) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) + if (self->fut_result == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_result); + return Py_NewRef(self->fut_result); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._exception +[clinic start generated code]*/ + static PyObject * -FutureObj_get_exception(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__exception_get_impl(FutureObj *self) +/*[clinic end generated code: output=32f2c93b9e021a9b input=1828a1fcac929710]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) - if (fut->fut_exception == NULL) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) + if (self->fut_exception == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_exception); + return Py_NewRef(self->fut_exception); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._source_traceback +[clinic start generated code]*/ + static PyObject * -FutureObj_get_source_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__source_traceback_get_impl(FutureObj *self) +/*[clinic end generated code: output=d4f12b09af22f61b input=3c831fbde5da90d0]*/ { - if (!future_is_alive(fut) || fut->fut_source_tb == NULL) { + if (!future_is_alive(self) || self->fut_source_tb == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_source_tb); + return Py_NewRef(self->fut_source_tb); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._cancel_message +[clinic start generated code]*/ + static PyObject * -FutureObj_get_cancel_message(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__cancel_message_get_impl(FutureObj *self) +/*[clinic end generated code: output=52ef6444f92cedac input=54c12c67082e4eea]*/ { - if (fut->fut_cancel_msg == NULL) { + if (self->fut_cancel_msg == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_cancel_msg); + return Py_NewRef(self->fut_cancel_msg); } +/*[clinic input] +@critical_section +@setter +_asyncio.Future._cancel_message +[clinic start generated code]*/ + static int -FutureObj_set_cancel_message(FutureObj *fut, PyObject *msg, - void *Py_UNUSED(ignored)) +_asyncio_Future__cancel_message_set_impl(FutureObj *self, PyObject *value) +/*[clinic end generated code: output=0854b2f77bff2209 input=f461d17f2d891fad]*/ { - if (msg == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - Py_INCREF(msg); - Py_XSETREF(fut->fut_cancel_msg, msg); + Py_INCREF(value); + Py_XSETREF(self->fut_cancel_msg, value); return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._state +[clinic start generated code]*/ + static PyObject * -FutureObj_get_state(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__state_get_impl(FutureObj *self) +/*[clinic end generated code: output=622f560a3fa69c63 input=7c5ad023a93423ff]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *ret = NULL; - ENSURE_FUTURE_ALIVE(state, fut) + ENSURE_FUTURE_ALIVE(state, self) - switch (fut->fut_state) { + switch (self->fut_state) { case STATE_PENDING: ret = &_Py_ID(PENDING); break; @@ -1364,6 +1475,7 @@ FutureObj_repr(FutureObj *fut) } /*[clinic input] +@critical_section _asyncio.Future._make_cancelled_error Create the CancelledError to raise if the Future is cancelled. @@ -1374,7 +1486,7 @@ it erases the context exception value. static PyObject * _asyncio_Future__make_cancelled_error_impl(FutureObj *self) -/*[clinic end generated code: output=a5df276f6c1213de input=ac6effe4ba795ecc]*/ +/*[clinic end generated code: output=a5df276f6c1213de input=ccb90df8c3c18bcd]*/ { asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); return create_cancelled_error(state, self); @@ -1455,23 +1567,16 @@ static PyMethodDef FutureType_methods[] = { {NULL, NULL} /* Sentinel */ }; -#define FUTURE_COMMON_GETSETLIST \ - {"_state", (getter)FutureObj_get_state, NULL, NULL}, \ - {"_asyncio_future_blocking", (getter)FutureObj_get_blocking, \ - (setter)FutureObj_set_blocking, NULL}, \ - {"_loop", (getter)FutureObj_get_loop, NULL, NULL}, \ - {"_callbacks", (getter)FutureObj_get_callbacks, NULL, NULL}, \ - {"_result", (getter)FutureObj_get_result, NULL, NULL}, \ - {"_exception", (getter)FutureObj_get_exception, NULL, NULL}, \ - {"_log_traceback", (getter)FutureObj_get_log_traceback, \ - (setter)FutureObj_set_log_traceback, NULL}, \ - {"_source_traceback", (getter)FutureObj_get_source_traceback, \ - NULL, NULL}, \ - {"_cancel_message", (getter)FutureObj_get_cancel_message, \ - (setter)FutureObj_set_cancel_message, NULL}, - static PyGetSetDef FutureType_getsetlist[] = { - FUTURE_COMMON_GETSETLIST + _ASYNCIO_FUTURE__STATE_GETSETDEF + _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF + _ASYNCIO_FUTURE__LOOP_GETSETDEF + _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF + _ASYNCIO_FUTURE__RESULT_GETSETDEF + _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF + _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF + _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF + _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {NULL} /* Sentinel */ }; @@ -1550,19 +1655,13 @@ FutureIter_dealloc(futureiterobject *it) } static PySendResult -FutureIter_am_send(futureiterobject *it, - PyObject *Py_UNUSED(arg), - PyObject **result) +FutureIter_am_send_lock_held(futureiterobject *it, PyObject **result) { - /* arg is unused, see the comment on FutureIter_send for clarification */ - PyObject *res; FutureObj *fut = it->future; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); *result = NULL; - if (fut == NULL) { - return PYGEN_ERROR; - } if (fut->fut_state == STATE_PENDING) { if (!fut->fut_blocking) { @@ -1575,18 +1674,29 @@ FutureIter_am_send(futureiterobject *it, return PYGEN_ERROR; } - it->future = NULL; res = _asyncio_Future_result_impl(fut); if (res != NULL) { - Py_DECREF(fut); *result = res; return PYGEN_RETURN; } - Py_DECREF(fut); return PYGEN_ERROR; } +static PySendResult +FutureIter_am_send(futureiterobject *it, + PyObject *Py_UNUSED(arg), + PyObject **result) +{ + /* arg is unused, see the comment on FutureIter_send for clarification */ + PySendResult res; + Py_BEGIN_CRITICAL_SECTION(it->future); + res = FutureIter_am_send_lock_held(it, result); + Py_END_CRITICAL_SECTION(); + return res; +} + + static PyObject * FutureIter_iternext(futureiterobject *it) { @@ -1807,7 +1917,11 @@ TaskStepMethWrapper_call(TaskStepMethWrapper *o, return NULL; } asyncio_state *state = get_asyncio_state_by_def((PyObject *)o); - return task_step(state, o->sw_task, o->sw_arg); + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(o->sw_task); + res = task_step(state, o->sw_task, o->sw_arg); + Py_END_CRITICAL_SECTION(); + return res; } static int @@ -2150,10 +2264,17 @@ TaskObj_traverse(TaskObj *task, visitproc visit, void *arg) return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Task._log_destroy_pending +[clinic start generated code]*/ + static PyObject * -TaskObj_get_log_destroy_pending(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__log_destroy_pending_get_impl(TaskObj *self) +/*[clinic end generated code: output=e6c2a47d029ac93b input=17127298cd4c720b]*/ { - if (task->task_log_destroy_pending) { + if (self->task_log_destroy_pending) { Py_RETURN_TRUE; } else { @@ -2161,25 +2282,40 @@ TaskObj_get_log_destroy_pending(TaskObj *task, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@setter +_asyncio.Task._log_destroy_pending +[clinic start generated code]*/ + static int -TaskObj_set_log_destroy_pending(TaskObj *task, PyObject *val, void *Py_UNUSED(ignored)) +_asyncio_Task__log_destroy_pending_set_impl(TaskObj *self, PyObject *value) +/*[clinic end generated code: output=7ebc030bb92ec5ce input=49b759c97d1216a4]*/ { - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - int is_true = PyObject_IsTrue(val); + int is_true = PyObject_IsTrue(value); if (is_true < 0) { return -1; } - task->task_log_destroy_pending = is_true; + self->task_log_destroy_pending = is_true; return 0; } + +/*[clinic input] +@critical_section +@getter +_asyncio.Task._must_cancel +[clinic start generated code]*/ + static PyObject * -TaskObj_get_must_cancel(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__must_cancel_get_impl(TaskObj *self) +/*[clinic end generated code: output=70e79b900996c363 input=2d04529fb23feedf]*/ { - if (task->task_must_cancel) { + if (self->task_must_cancel) { Py_RETURN_TRUE; } else { @@ -2187,21 +2323,36 @@ TaskObj_get_must_cancel(TaskObj *task, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@getter +_asyncio.Task._coro +[clinic start generated code]*/ + static PyObject * -TaskObj_get_coro(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__coro_get_impl(TaskObj *self) +/*[clinic end generated code: output=a2726012ab5fd531 input=323c31a272020624]*/ { - if (task->task_coro) { - return Py_NewRef(task->task_coro); + if (self->task_coro) { + return Py_NewRef(self->task_coro); } Py_RETURN_NONE; } + +/*[clinic input] +@critical_section +@getter +_asyncio.Task._fut_waiter +[clinic start generated code]*/ + static PyObject * -TaskObj_get_fut_waiter(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__fut_waiter_get_impl(TaskObj *self) +/*[clinic end generated code: output=c4f966b847fefcdf input=4d1005d725e72db7]*/ { - if (task->task_fut_waiter) { - return Py_NewRef(task->task_fut_waiter); + if (self->task_fut_waiter) { + return Py_NewRef(self->task_fut_waiter); } Py_RETURN_NONE; @@ -2217,6 +2368,7 @@ TaskObj_repr(TaskObj *task) /*[clinic input] +@critical_section _asyncio.Task._make_cancelled_error Create the CancelledError to raise if the Task is cancelled. @@ -2227,7 +2379,7 @@ it erases the context exception value. static PyObject * _asyncio_Task__make_cancelled_error_impl(TaskObj *self) -/*[clinic end generated code: output=55a819e8b4276fab input=52c0e32de8e2f840]*/ +/*[clinic end generated code: output=55a819e8b4276fab input=2d3213be0cb02390]*/ { FutureObj *fut = (FutureObj*)self; return _asyncio_Future__make_cancelled_error_impl(fut); @@ -2235,6 +2387,7 @@ _asyncio_Task__make_cancelled_error_impl(TaskObj *self) /*[clinic input] +@critical_section _asyncio.Task.cancel msg: object = None @@ -2263,7 +2416,7 @@ This also increases the task's count of cancellation requests. static PyObject * _asyncio_Task_cancel_impl(TaskObj *self, PyObject *msg) -/*[clinic end generated code: output=c66b60d41c74f9f1 input=7bb51bf25974c783]*/ +/*[clinic end generated code: output=c66b60d41c74f9f1 input=6125d45b9a6a5abd]*/ { self->task_log_tb = 0; @@ -2308,6 +2461,7 @@ _asyncio_Task_cancel_impl(TaskObj *self, PyObject *msg) } /*[clinic input] +@critical_section _asyncio.Task.cancelling Return the count of the task's cancellation requests. @@ -2318,13 +2472,14 @@ and may be decremented using .uncancel(). static PyObject * _asyncio_Task_cancelling_impl(TaskObj *self) -/*[clinic end generated code: output=803b3af96f917d7e input=b625224d310cbb17]*/ +/*[clinic end generated code: output=803b3af96f917d7e input=5ef89b1b38f080ee]*/ /*[clinic end generated code]*/ { return PyLong_FromLong(self->task_num_cancels_requested); } /*[clinic input] +@critical_section _asyncio.Task.uncancel Decrement the task's count of cancellation requests. @@ -2337,7 +2492,7 @@ Returns the remaining number of cancellation requests. static PyObject * _asyncio_Task_uncancel_impl(TaskObj *self) -/*[clinic end generated code: output=58184d236a817d3c input=68f81a4b90b46be2]*/ +/*[clinic end generated code: output=58184d236a817d3c input=cb3220b0e5afd61d]*/ /*[clinic end generated code]*/ { if (self->task_num_cancels_requested > 0) { @@ -2451,12 +2606,13 @@ _asyncio_Task_set_exception(TaskObj *self, PyObject *exception) } /*[clinic input] +@critical_section _asyncio.Task.get_coro [clinic start generated code]*/ static PyObject * _asyncio_Task_get_coro_impl(TaskObj *self) -/*[clinic end generated code: output=bcac27c8cc6c8073 input=d2e8606c42a7b403]*/ +/*[clinic end generated code: output=bcac27c8cc6c8073 input=a47f81427e39fe0c]*/ { if (self->task_coro) { return Py_NewRef(self->task_coro); @@ -2477,12 +2633,13 @@ _asyncio_Task_get_context_impl(TaskObj *self) } /*[clinic input] +@critical_section _asyncio.Task.get_name [clinic start generated code]*/ static PyObject * _asyncio_Task_get_name_impl(TaskObj *self) -/*[clinic end generated code: output=0ecf1570c3b37a8f input=a4a6595d12f4f0f8]*/ +/*[clinic end generated code: output=0ecf1570c3b37a8f input=92a8f30c85034249]*/ { if (self->task_name) { if (PyLong_CheckExact(self->task_name)) { @@ -2499,6 +2656,7 @@ _asyncio_Task_get_name_impl(TaskObj *self) } /*[clinic input] +@critical_section _asyncio.Task.set_name value: object @@ -2506,8 +2664,8 @@ _asyncio.Task.set_name [clinic start generated code]*/ static PyObject * -_asyncio_Task_set_name(TaskObj *self, PyObject *value) -/*[clinic end generated code: output=138a8d51e32057d6 input=a8359b6e65f8fd31]*/ +_asyncio_Task_set_name_impl(TaskObj *self, PyObject *value) +/*[clinic end generated code: output=f88ff4c0d64a9a6f input=e8d400ad64bad799]*/ { if (!PyUnicode_CheckExact(value)) { value = PyObject_Str(value); @@ -2618,12 +2776,10 @@ static PyMethodDef TaskType_methods[] = { }; static PyGetSetDef TaskType_getsetlist[] = { - FUTURE_COMMON_GETSETLIST - {"_log_destroy_pending", (getter)TaskObj_get_log_destroy_pending, - (setter)TaskObj_set_log_destroy_pending, NULL}, - {"_must_cancel", (getter)TaskObj_get_must_cancel, NULL, NULL}, - {"_coro", (getter)TaskObj_get_coro, NULL, NULL}, - {"_fut_waiter", (getter)TaskObj_get_fut_waiter, NULL, NULL}, + _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF + _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF + _ASYNCIO_TASK__CORO_GETSETDEF + _ASYNCIO_TASK__FUT_WAITER_GETSETDEF {NULL} /* Sentinel */ }; @@ -2738,6 +2894,8 @@ gen_status_from_result(PyObject **result) static PyObject * task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(task); + int clear_exc = 0; PyObject *result = NULL; PyObject *coro; @@ -2867,6 +3025,8 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) static PyObject * task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *result) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(task); + int res; PyObject *o; @@ -2897,8 +3057,10 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (wrapper == NULL) { goto fail; } + Py_BEGIN_CRITICAL_SECTION(result); tmp = future_add_done_callback(state, (FutureObj*)result, wrapper, task->task_context); + Py_END_CRITICAL_SECTION(); Py_DECREF(wrapper); if (tmp == NULL) { goto fail; @@ -3140,7 +3302,10 @@ task_eager_start(asyncio_state *state, TaskObj *task) int retval = 0; - PyObject *stepres = task_step_impl(state, task, NULL); + PyObject *stepres; + Py_BEGIN_CRITICAL_SECTION(task); + stepres = task_step_impl(state, task, NULL); + Py_END_CRITICAL_SECTION(); if (stepres == NULL) { PyObject *exc = PyErr_GetRaisedException(); _PyErr_ChainExceptions1(exc); @@ -3177,16 +3342,20 @@ task_eager_start(asyncio_state *state, TaskObj *task) } static PyObject * -task_wakeup(TaskObj *task, PyObject *o) +task_wakeup_lock_held(TaskObj *task, PyObject *o) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(task); + PyObject *result; assert(o); asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); if (Future_CheckExact(state, o) || Task_CheckExact(state, o)) { PyObject *fut_result = NULL; - int res = future_get_result(state, (FutureObj*)o, &fut_result); - + int res; + Py_BEGIN_CRITICAL_SECTION(o); + res = future_get_result(state, (FutureObj*)o, &fut_result); + Py_END_CRITICAL_SECTION(); switch(res) { case -1: assert(fut_result == NULL); @@ -3220,6 +3389,16 @@ task_wakeup(TaskObj *task, PyObject *o) return result; } +static PyObject * +task_wakeup(TaskObj *task, PyObject *o) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(task); + res = task_wakeup_lock_held(task, o); + Py_END_CRITICAL_SECTION(); + return res; +} + /*********************** Functions **************************/ @@ -3564,62 +3743,41 @@ static PyObject * _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) /*[clinic end generated code: output=0e107cbb7f72aa7b input=43a1b423c2d95bfa]*/ { - asyncio_state *state = get_asyncio_state(module); - PyObject *tasks = PySet_New(NULL); - if (tasks == NULL) { - return NULL; - } if (loop == Py_None) { loop = _asyncio_get_running_loop_impl(module); if (loop == NULL) { - Py_DECREF(tasks); return NULL; } } else { Py_INCREF(loop); } - // First add eager tasks to the set so that we don't miss + // First add eager tasks to the list so that we don't miss // any tasks which graduates from eager to non-eager - PyObject *eager_iter = PyObject_GetIter(state->eager_tasks); - if (eager_iter == NULL) { - Py_DECREF(tasks); + // We first add all the tasks to `tasks` list and then filter + // out the tasks which are done and return it as a set. + PyObject *tasks = PyList_New(0); + if (tasks == NULL) { Py_DECREF(loop); return NULL; } - PyObject *item; - while ((item = PyIter_Next(eager_iter)) != NULL) { - if (add_one_task(state, tasks, item, loop) < 0) { - Py_DECREF(tasks); - Py_DECREF(loop); - Py_DECREF(item); - Py_DECREF(eager_iter); - return NULL; - } - Py_DECREF(item); - } - Py_DECREF(eager_iter); - - if (PyErr_Occurred()) { + if (PyList_Extend(tasks, state->eager_tasks) < 0) { Py_DECREF(tasks); Py_DECREF(loop); return NULL; } - int err = 0; ASYNCIO_STATE_LOCK(state); struct llist_node *node; + llist_for_each_safe(node, &state->asyncio_tasks_head) { TaskObj *task = llist_data(node, TaskObj, task_node); - Py_INCREF(task); - if (add_one_task(state, tasks, (PyObject *)task, loop) < 0) { - Py_DECREF(task); + if (PyList_Append(tasks, (PyObject *)task) < 0) { Py_DECREF(tasks); Py_DECREF(loop); err = 1; break; } - Py_DECREF(task); } ASYNCIO_STATE_UNLOCK(state); if (err) { @@ -3631,8 +3789,9 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) Py_DECREF(loop); return NULL; } + PyObject *item; while ((item = PyIter_Next(scheduled_iter)) != NULL) { - if (add_one_task(state, tasks, item, loop) < 0) { + if (PyList_Append(tasks, item) < 0) { Py_DECREF(tasks); Py_DECREF(loop); Py_DECREF(item); @@ -3642,14 +3801,27 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) Py_DECREF(item); } Py_DECREF(scheduled_iter); - Py_DECREF(loop); - - if (PyErr_Occurred()) { + // All the tasks are now in the list, now filter the tasks which are done + PyObject *res = PySet_New(NULL); + if (res == NULL) { Py_DECREF(tasks); + Py_DECREF(loop); return NULL; } - return tasks; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(tasks); i++) { + PyObject *task = PyList_GET_ITEM(tasks, i); + if (add_one_task(state, res, task, loop) < 0) { + Py_DECREF(res); + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + } + + Py_DECREF(tasks); + Py_DECREF(loop); + return res; } static int diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 32045804c35004..3a37cdd9b5fa83 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_asyncio_Future___init____doc__, @@ -98,7 +99,13 @@ _asyncio_Future_result_impl(FutureObj *self); static PyObject * _asyncio_Future_result(FutureObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future_result_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_result_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future_exception__doc__, @@ -121,11 +128,18 @@ _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_exception(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "exception() takes no arguments"); - return NULL; + goto exit; } - return _asyncio_Future_exception_impl(self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_exception_impl(self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_asyncio_Future_set_result__doc__, @@ -170,7 +184,9 @@ _asyncio_Future_set_result(FutureObj *self, PyTypeObject *cls, PyObject *const * goto exit; } result = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Future_set_result_impl(self, cls, result); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -218,7 +234,9 @@ _asyncio_Future_set_exception(FutureObj *self, PyTypeObject *cls, PyObject *cons goto exit; } exception = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Future_set_exception_impl(self, cls, exception); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -286,7 +304,9 @@ _asyncio_Future_add_done_callback(FutureObj *self, PyTypeObject *cls, PyObject * } context = args[1]; skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Future_add_done_callback_impl(self, cls, fn, context); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -333,7 +353,9 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyTypeObject *cls, PyObjec goto exit; } fn = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Future_remove_done_callback_impl(self, cls, fn); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -399,7 +421,9 @@ _asyncio_Future_cancel(FutureObj *self, PyTypeObject *cls, PyObject *const *args } msg = args[0]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Future_cancel_impl(self, cls, msg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -420,7 +444,13 @@ _asyncio_Future_cancelled_impl(FutureObj *self); static PyObject * _asyncio_Future_cancelled(FutureObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future_cancelled_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_cancelled_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future_done__doc__, @@ -441,7 +471,13 @@ _asyncio_Future_done_impl(FutureObj *self); static PyObject * _asyncio_Future_done(FutureObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future_done_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_done_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future_get_loop__doc__, @@ -459,11 +495,319 @@ _asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_get_loop(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_loop() takes no arguments"); - return NULL; + goto exit; } - return _asyncio_Future_get_loop_impl(self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_get_loop_impl(self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#if !defined(_asyncio_Future__asyncio_future_blocking_DOCSTR) +# define _asyncio_Future__asyncio_future_blocking_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF) +# undef _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", (getter)_asyncio_Future__asyncio_future_blocking_get, (setter)_asyncio_Future__asyncio_future_blocking_set, _asyncio_Future__asyncio_future_blocking_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", (getter)_asyncio_Future__asyncio_future_blocking_get, NULL, _asyncio_Future__asyncio_future_blocking_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__asyncio_future_blocking_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__asyncio_future_blocking_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__asyncio_future_blocking_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__asyncio_future_blocking_DOCSTR) +# define _asyncio_Future__asyncio_future_blocking_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF) +# undef _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", (getter)_asyncio_Future__asyncio_future_blocking_get, (setter)_asyncio_Future__asyncio_future_blocking_set, _asyncio_Future__asyncio_future_blocking_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", NULL, (setter)_asyncio_Future__asyncio_future_blocking_set, NULL}, +#endif + +static int +_asyncio_Future__asyncio_future_blocking_set_impl(FutureObj *self, + PyObject *value); + +static int +_asyncio_Future__asyncio_future_blocking_set(FutureObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__asyncio_future_blocking_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__log_traceback_DOCSTR) +# define _asyncio_Future__log_traceback_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF) +# undef _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", (getter)_asyncio_Future__log_traceback_get, (setter)_asyncio_Future__log_traceback_set, _asyncio_Future__log_traceback_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", (getter)_asyncio_Future__log_traceback_get, NULL, _asyncio_Future__log_traceback_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__log_traceback_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__log_traceback_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__log_traceback_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__log_traceback_DOCSTR) +# define _asyncio_Future__log_traceback_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF) +# undef _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", (getter)_asyncio_Future__log_traceback_get, (setter)_asyncio_Future__log_traceback_set, _asyncio_Future__log_traceback_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", NULL, (setter)_asyncio_Future__log_traceback_set, NULL}, +#endif + +static int +_asyncio_Future__log_traceback_set_impl(FutureObj *self, PyObject *value); + +static int +_asyncio_Future__log_traceback_set(FutureObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__log_traceback_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__loop_DOCSTR) +# define _asyncio_Future__loop_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__LOOP_GETSETDEF) +# undef _ASYNCIO_FUTURE__LOOP_GETSETDEF +# define _ASYNCIO_FUTURE__LOOP_GETSETDEF {"_loop", (getter)_asyncio_Future__loop_get, (setter)_asyncio_Future__loop_set, _asyncio_Future__loop_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__LOOP_GETSETDEF {"_loop", (getter)_asyncio_Future__loop_get, NULL, _asyncio_Future__loop_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__loop_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__loop_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__loop_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__callbacks_DOCSTR) +# define _asyncio_Future__callbacks_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__CALLBACKS_GETSETDEF) +# undef _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF +# define _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF {"_callbacks", (getter)_asyncio_Future__callbacks_get, (setter)_asyncio_Future__callbacks_set, _asyncio_Future__callbacks_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF {"_callbacks", (getter)_asyncio_Future__callbacks_get, NULL, _asyncio_Future__callbacks_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__callbacks_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__callbacks_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__callbacks_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__result_DOCSTR) +# define _asyncio_Future__result_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__RESULT_GETSETDEF) +# undef _ASYNCIO_FUTURE__RESULT_GETSETDEF +# define _ASYNCIO_FUTURE__RESULT_GETSETDEF {"_result", (getter)_asyncio_Future__result_get, (setter)_asyncio_Future__result_set, _asyncio_Future__result_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__RESULT_GETSETDEF {"_result", (getter)_asyncio_Future__result_get, NULL, _asyncio_Future__result_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__result_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__result_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__result_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__exception_DOCSTR) +# define _asyncio_Future__exception_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__EXCEPTION_GETSETDEF) +# undef _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF +# define _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF {"_exception", (getter)_asyncio_Future__exception_get, (setter)_asyncio_Future__exception_set, _asyncio_Future__exception_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF {"_exception", (getter)_asyncio_Future__exception_get, NULL, _asyncio_Future__exception_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__exception_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__exception_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__exception_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__source_traceback_DOCSTR) +# define _asyncio_Future__source_traceback_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF) +# undef _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF +# define _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF {"_source_traceback", (getter)_asyncio_Future__source_traceback_get, (setter)_asyncio_Future__source_traceback_set, _asyncio_Future__source_traceback_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF {"_source_traceback", (getter)_asyncio_Future__source_traceback_get, NULL, _asyncio_Future__source_traceback_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__source_traceback_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__source_traceback_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__source_traceback_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__cancel_message_DOCSTR) +# define _asyncio_Future__cancel_message_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF) +# undef _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", (getter)_asyncio_Future__cancel_message_get, (setter)_asyncio_Future__cancel_message_set, _asyncio_Future__cancel_message_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", (getter)_asyncio_Future__cancel_message_get, NULL, _asyncio_Future__cancel_message_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__cancel_message_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__cancel_message_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__cancel_message_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__cancel_message_DOCSTR) +# define _asyncio_Future__cancel_message_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF) +# undef _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", (getter)_asyncio_Future__cancel_message_get, (setter)_asyncio_Future__cancel_message_set, _asyncio_Future__cancel_message_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", NULL, (setter)_asyncio_Future__cancel_message_set, NULL}, +#endif + +static int +_asyncio_Future__cancel_message_set_impl(FutureObj *self, PyObject *value); + +static int +_asyncio_Future__cancel_message_set(FutureObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__cancel_message_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__state_DOCSTR) +# define _asyncio_Future__state_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__STATE_GETSETDEF) +# undef _ASYNCIO_FUTURE__STATE_GETSETDEF +# define _ASYNCIO_FUTURE__STATE_GETSETDEF {"_state", (getter)_asyncio_Future__state_get, (setter)_asyncio_Future__state_set, _asyncio_Future__state_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__STATE_GETSETDEF {"_state", (getter)_asyncio_Future__state_get, NULL, _asyncio_Future__state_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__state_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__state_get(FutureObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__state_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future__make_cancelled_error__doc__, @@ -484,7 +828,13 @@ _asyncio_Future__make_cancelled_error_impl(FutureObj *self); static PyObject * _asyncio_Future__make_cancelled_error(FutureObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future__make_cancelled_error_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__make_cancelled_error_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task___init____doc__, @@ -575,6 +925,131 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } +#if !defined(_asyncio_Task__log_destroy_pending_DOCSTR) +# define _asyncio_Task__log_destroy_pending_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF) +# undef _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", (getter)_asyncio_Task__log_destroy_pending_get, (setter)_asyncio_Task__log_destroy_pending_set, _asyncio_Task__log_destroy_pending_DOCSTR}, +#else +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", (getter)_asyncio_Task__log_destroy_pending_get, NULL, _asyncio_Task__log_destroy_pending_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__log_destroy_pending_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__log_destroy_pending_get(TaskObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__log_destroy_pending_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__log_destroy_pending_DOCSTR) +# define _asyncio_Task__log_destroy_pending_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF) +# undef _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", (getter)_asyncio_Task__log_destroy_pending_get, (setter)_asyncio_Task__log_destroy_pending_set, _asyncio_Task__log_destroy_pending_DOCSTR}, +#else +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", NULL, (setter)_asyncio_Task__log_destroy_pending_set, NULL}, +#endif + +static int +_asyncio_Task__log_destroy_pending_set_impl(TaskObj *self, PyObject *value); + +static int +_asyncio_Task__log_destroy_pending_set(TaskObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__log_destroy_pending_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__must_cancel_DOCSTR) +# define _asyncio_Task__must_cancel_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__MUST_CANCEL_GETSETDEF) +# undef _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF +# define _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF {"_must_cancel", (getter)_asyncio_Task__must_cancel_get, (setter)_asyncio_Task__must_cancel_set, _asyncio_Task__must_cancel_DOCSTR}, +#else +# define _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF {"_must_cancel", (getter)_asyncio_Task__must_cancel_get, NULL, _asyncio_Task__must_cancel_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__must_cancel_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__must_cancel_get(TaskObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__must_cancel_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__coro_DOCSTR) +# define _asyncio_Task__coro_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__CORO_GETSETDEF) +# undef _ASYNCIO_TASK__CORO_GETSETDEF +# define _ASYNCIO_TASK__CORO_GETSETDEF {"_coro", (getter)_asyncio_Task__coro_get, (setter)_asyncio_Task__coro_set, _asyncio_Task__coro_DOCSTR}, +#else +# define _ASYNCIO_TASK__CORO_GETSETDEF {"_coro", (getter)_asyncio_Task__coro_get, NULL, _asyncio_Task__coro_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__coro_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__coro_get(TaskObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__coro_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__fut_waiter_DOCSTR) +# define _asyncio_Task__fut_waiter_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__FUT_WAITER_GETSETDEF) +# undef _ASYNCIO_TASK__FUT_WAITER_GETSETDEF +# define _ASYNCIO_TASK__FUT_WAITER_GETSETDEF {"_fut_waiter", (getter)_asyncio_Task__fut_waiter_get, (setter)_asyncio_Task__fut_waiter_set, _asyncio_Task__fut_waiter_DOCSTR}, +#else +# define _ASYNCIO_TASK__FUT_WAITER_GETSETDEF {"_fut_waiter", (getter)_asyncio_Task__fut_waiter_get, NULL, _asyncio_Task__fut_waiter_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__fut_waiter_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__fut_waiter_get(TaskObj *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__fut_waiter_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_asyncio_Task__make_cancelled_error__doc__, "_make_cancelled_error($self, /)\n" "--\n" @@ -593,7 +1068,13 @@ _asyncio_Task__make_cancelled_error_impl(TaskObj *self); static PyObject * _asyncio_Task__make_cancelled_error(TaskObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task__make_cancelled_error_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__make_cancelled_error_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_cancel__doc__, @@ -670,7 +1151,9 @@ _asyncio_Task_cancel(TaskObj *self, PyObject *const *args, Py_ssize_t nargs, PyO } msg = args[0]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _asyncio_Task_cancel_impl(self, msg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -694,7 +1177,13 @@ _asyncio_Task_cancelling_impl(TaskObj *self); static PyObject * _asyncio_Task_cancelling(TaskObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_cancelling_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_cancelling_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_uncancel__doc__, @@ -717,7 +1206,13 @@ _asyncio_Task_uncancel_impl(TaskObj *self); static PyObject * _asyncio_Task_uncancel(TaskObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_uncancel_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_uncancel_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_get_stack__doc__, @@ -905,7 +1400,13 @@ _asyncio_Task_get_coro_impl(TaskObj *self); static PyObject * _asyncio_Task_get_coro(TaskObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_get_coro_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_get_coro_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_get_context__doc__, @@ -939,7 +1440,13 @@ _asyncio_Task_get_name_impl(TaskObj *self); static PyObject * _asyncio_Task_get_name(TaskObj *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_get_name_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_get_name_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_set_name__doc__, @@ -950,6 +1457,21 @@ PyDoc_STRVAR(_asyncio_Task_set_name__doc__, #define _ASYNCIO_TASK_SET_NAME_METHODDEF \ {"set_name", (PyCFunction)_asyncio_Task_set_name, METH_O, _asyncio_Task_set_name__doc__}, +static PyObject * +_asyncio_Task_set_name_impl(TaskObj *self, PyObject *value); + +static PyObject * +_asyncio_Task_set_name(TaskObj *self, PyObject *value) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_set_name_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_asyncio__get_running_loop__doc__, "_get_running_loop($module, /)\n" "--\n" @@ -1566,4 +2088,4 @@ _asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=e5d95a0ec229ffcd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=408e156476ced07f input=a9049054013a1b77]*/ From 03ede5afe2d10c04e05f159fd353ee5869ae2cdb Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sat, 4 Jan 2025 13:31:01 +0300 Subject: [PATCH 362/427] Add `check-readthedocs` pre-commit hook (#128453) --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 107f3b255735f4..af6accd89b5bd4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,6 +55,7 @@ repos: hooks: - id: check-dependabot - id: check-github-workflows + - id: check-readthedocs - repo: https://github.com/rhysd/actionlint rev: v1.7.4 From a4e773c540cfd3a9c2bb3b5033d2f79ef50962c8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 4 Jan 2025 11:46:04 +0100 Subject: [PATCH 363/427] gh-128152: Argument Clinic: ignore pre-processor directives inside C comments (#128464) --- Lib/test/test_clinic.py | 10 ++++++++++ .../2025-01-03-23-51-07.gh-issue-128152.IhzElS.rst | 2 ++ Tools/clinic/libclinic/cpp.py | 3 +++ 3 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-01-03-23-51-07.gh-issue-128152.IhzElS.rst diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 11054963b6ff03..b45b9ee89ee3de 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -740,6 +740,16 @@ def test_cloned_forced_text_signature_illegal(self): err = "Cannot use @text_signature when cloning a function" self.expect_failure(block, err, lineno=11) + def test_ignore_preprocessor_in_comments(self): + for dsl in "clinic", "python": + raw = dedent(f"""\ + /*[{dsl} input] + # CPP directives, valid or not, should be ignored in C comments. + # + [{dsl} start generated code]*/ + """) + self.clinic.parse(raw) + class ParseFileUnitTest(TestCase): def expect_parsing_failure( diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-01-03-23-51-07.gh-issue-128152.IhzElS.rst b/Misc/NEWS.d/next/Tools-Demos/2025-01-03-23-51-07.gh-issue-128152.IhzElS.rst new file mode 100644 index 00000000000000..9657e138e9911b --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-01-03-23-51-07.gh-issue-128152.IhzElS.rst @@ -0,0 +1,2 @@ +Fix a bug where Argument Clinic's C pre-processor parser tried to parse +pre-processor directives inside C comments. Patch by Erlend Aasland. diff --git a/Tools/clinic/libclinic/cpp.py b/Tools/clinic/libclinic/cpp.py index e115d65a88e1b6..3cfe99b712641d 100644 --- a/Tools/clinic/libclinic/cpp.py +++ b/Tools/clinic/libclinic/cpp.py @@ -132,6 +132,9 @@ def pop_stack() -> TokenAndCondition: if line_comment: line = before.rstrip() + if self.in_comment: + return + if not line.startswith('#'): return From 95352dcb9321423d0606ae0b01524ad61c2c2ec1 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 4 Jan 2025 12:53:51 +0000 Subject: [PATCH 364/427] GH-127381: pathlib ABCs: remove `PathBase.move()` and `move_into()` (#128337) These methods combine `_delete()` and `copy()`, but `_delete()` isn't part of the public interface, and it's unlikely to be added until the pathlib ABCs are made official, or perhaps even later. --- Lib/pathlib/_abc.py | 27 ----- Lib/pathlib/_local.py | 38 +++++-- Lib/test/test_pathlib/test_pathlib.py | 118 ++++++++++++++++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 118 ---------------------- 4 files changed, 148 insertions(+), 153 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index e6ff3fe1187512..7de2bb066f8f99 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -573,30 +573,3 @@ def copy_into(self, target_dir, *, follow_symlinks=True, return self.copy(target, follow_symlinks=follow_symlinks, dirs_exist_ok=dirs_exist_ok, preserve_metadata=preserve_metadata) - - def _delete(self): - """ - Delete this file or directory (including all sub-directories). - """ - raise NotImplementedError - - def move(self, target): - """ - Recursively move this file or directory tree to the given destination. - """ - target = self.copy(target, follow_symlinks=False, preserve_metadata=True) - self._delete() - return target - - def move_into(self, target_dir): - """ - Move this file or directory tree into the given existing directory. - """ - name = self.name - if not name: - raise ValueError(f"{self!r} has an empty name") - elif isinstance(target_dir, PathBase): - target = target_dir / name - else: - target = self.with_segments(target_dir, name) - return self.move(target) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index c5721a69d00470..1da85ddea24376 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -1128,16 +1128,38 @@ def move(self, target): """ Recursively move this file or directory tree to the given destination. """ - if not isinstance(target, PathBase): - target = self.with_segments(target) - target.copy._ensure_different_file(self) + # Use os.replace() if the target is os.PathLike and on the same FS. try: - return self.replace(target) - except OSError as err: - if err.errno != EXDEV: - raise + target_str = os.fspath(target) + except TypeError: + pass + else: + if not isinstance(target, PathBase): + target = self.with_segments(target_str) + target.copy._ensure_different_file(self) + try: + os.replace(self, target_str) + return target + except OSError as err: + if err.errno != EXDEV: + raise # Fall back to copy+delete. - return PathBase.move(self, target) + target = self.copy(target, follow_symlinks=False, preserve_metadata=True) + self._delete() + return target + + def move_into(self, target_dir): + """ + Move this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif isinstance(target_dir, PathBase): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.move(target) if hasattr(os, "symlink"): def symlink_to(self, target, target_is_directory=False): diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index d13daf8ac8cb07..a67a1c531630a1 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1423,26 +1423,97 @@ def test_move_dangling_symlink(self): self.assertTrue(target.is_symlink()) self.assertEqual(source_readlink, target.readlink()) + def test_move_file(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target = base / 'fileA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.exists()) + self.assertEqual(source_text, target.read_text()) + @patch_replace def test_move_file_other_fs(self): self.test_move_file() + def test_move_file_to_file(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target = base / 'dirB' / 'fileB' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.exists()) + self.assertEqual(source_text, target.read_text()) + @patch_replace def test_move_file_to_file_other_fs(self): self.test_move_file_to_file() + def test_move_file_to_dir(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirB' + self.assertRaises(OSError, source.move, target) + @patch_replace def test_move_file_to_dir_other_fs(self): self.test_move_file_to_dir() + def test_move_file_to_itself(self): + base = self.cls(self.base) + source = base / 'fileA' + self.assertRaises(OSError, source.move, source) + + def test_move_dir(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + @patch_replace def test_move_dir_other_fs(self): self.test_move_dir() + def test_move_dir_to_dir(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirB' + self.assertRaises(OSError, source.move, target) + self.assertTrue(source.exists()) + self.assertTrue(target.exists()) + @patch_replace def test_move_dir_to_dir_other_fs(self): self.test_move_dir_to_dir() + def test_move_dir_to_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + self.assertRaises(OSError, source.move, source) + self.assertTrue(source.exists()) + + def test_move_dir_into_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC' / 'bar' + self.assertRaises(OSError, source.move, target) + self.assertTrue(source.exists()) + self.assertFalse(target.exists()) + @patch_replace def test_move_dir_into_itself_other_fs(self): self.test_move_dir_into_itself() @@ -1472,10 +1543,26 @@ def test_move_dir_symlink_to_itself_other_fs(self): def test_move_dangling_symlink_other_fs(self): self.test_move_dangling_symlink() + def test_move_into(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target_dir = base / 'dirA' + result = source.move_into(target_dir) + self.assertEqual(result, target_dir / 'fileA') + self.assertFalse(source.exists()) + self.assertTrue(result.exists()) + self.assertEqual(source_text, result.read_text()) + @patch_replace def test_move_into_other_os(self): self.test_move_into() + def test_move_into_empty_name(self): + source = self.cls('') + target_dir = self.base + self.assertRaises(ValueError, source.move_into, target_dir) + @patch_replace def test_move_into_empty_name_other_os(self): self.test_move_into_empty_name() @@ -1794,6 +1881,37 @@ def test_rmdir(self): self.assertFileNotFound(p.stat) self.assertFileNotFound(p.unlink) + def test_delete_file(self): + p = self.cls(self.base) / 'fileA' + p._delete() + self.assertFalse(p.exists()) + self.assertFileNotFound(p._delete) + + def test_delete_dir(self): + base = self.cls(self.base) + base.joinpath('dirA')._delete() + self.assertFalse(base.joinpath('dirA').exists()) + self.assertFalse(base.joinpath('dirA', 'linkC').exists( + follow_symlinks=False)) + base.joinpath('dirB')._delete() + self.assertFalse(base.joinpath('dirB').exists()) + self.assertFalse(base.joinpath('dirB', 'fileB').exists()) + self.assertFalse(base.joinpath('dirB', 'linkD').exists( + follow_symlinks=False)) + base.joinpath('dirC')._delete() + self.assertFalse(base.joinpath('dirC').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD', 'fileD').exists()) + self.assertFalse(base.joinpath('dirC', 'fileC').exists()) + self.assertFalse(base.joinpath('dirC', 'novel.txt').exists()) + + def test_delete_missing(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + # filename is guaranteed not to exist + filename = tmp / 'foo' + self.assertRaises(FileNotFoundError, filename._delete) + @needs_symlinks def test_delete_symlink(self): tmp = self.cls(self.base, 'delete') diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index d588442bd11785..0762f224fc9ac4 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1345,93 +1345,6 @@ def test_copy_into_empty_name(self): target_dir = self.base self.assertRaises(ValueError, source.copy_into, target_dir) - def test_move_file(self): - base = self.cls(self.base) - source = base / 'fileA' - source_text = source.read_text() - target = base / 'fileA_moved' - result = source.move(target) - self.assertEqual(result, target) - self.assertFalse(source.exists()) - self.assertTrue(target.exists()) - self.assertEqual(source_text, target.read_text()) - - def test_move_file_to_file(self): - base = self.cls(self.base) - source = base / 'fileA' - source_text = source.read_text() - target = base / 'dirB' / 'fileB' - result = source.move(target) - self.assertEqual(result, target) - self.assertFalse(source.exists()) - self.assertTrue(target.exists()) - self.assertEqual(source_text, target.read_text()) - - def test_move_file_to_dir(self): - base = self.cls(self.base) - source = base / 'fileA' - target = base / 'dirB' - self.assertRaises(OSError, source.move, target) - - def test_move_file_to_itself(self): - base = self.cls(self.base) - source = base / 'fileA' - self.assertRaises(OSError, source.move, source) - - def test_move_dir(self): - base = self.cls(self.base) - source = base / 'dirC' - target = base / 'dirC_moved' - result = source.move(target) - self.assertEqual(result, target) - self.assertFalse(source.exists()) - self.assertTrue(target.is_dir()) - self.assertTrue(target.joinpath('dirD').is_dir()) - self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) - self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), - "this is file D\n") - self.assertTrue(target.joinpath('fileC').is_file()) - self.assertTrue(target.joinpath('fileC').read_text(), - "this is file C\n") - - def test_move_dir_to_dir(self): - base = self.cls(self.base) - source = base / 'dirC' - target = base / 'dirB' - self.assertRaises(OSError, source.move, target) - self.assertTrue(source.exists()) - self.assertTrue(target.exists()) - - def test_move_dir_to_itself(self): - base = self.cls(self.base) - source = base / 'dirC' - self.assertRaises(OSError, source.move, source) - self.assertTrue(source.exists()) - - def test_move_dir_into_itself(self): - base = self.cls(self.base) - source = base / 'dirC' - target = base / 'dirC' / 'bar' - self.assertRaises(OSError, source.move, target) - self.assertTrue(source.exists()) - self.assertFalse(target.exists()) - - def test_move_into(self): - base = self.cls(self.base) - source = base / 'fileA' - source_text = source.read_text() - target_dir = base / 'dirA' - result = source.move_into(target_dir) - self.assertEqual(result, target_dir / 'fileA') - self.assertFalse(source.exists()) - self.assertTrue(result.exists()) - self.assertEqual(source_text, result.read_text()) - - def test_move_into_empty_name(self): - source = self.cls('') - target_dir = self.base - self.assertRaises(ValueError, source.move_into, target_dir) - def test_iterdir(self): P = self.cls p = P(self.base) @@ -1660,37 +1573,6 @@ def test_is_symlink(self): self.assertIs((P / 'linkA\udfff').is_file(), False) self.assertIs((P / 'linkA\x00').is_file(), False) - def test_delete_file(self): - p = self.cls(self.base) / 'fileA' - p._delete() - self.assertFalse(p.exists()) - self.assertFileNotFound(p._delete) - - def test_delete_dir(self): - base = self.cls(self.base) - base.joinpath('dirA')._delete() - self.assertFalse(base.joinpath('dirA').exists()) - self.assertFalse(base.joinpath('dirA', 'linkC').exists( - follow_symlinks=False)) - base.joinpath('dirB')._delete() - self.assertFalse(base.joinpath('dirB').exists()) - self.assertFalse(base.joinpath('dirB', 'fileB').exists()) - self.assertFalse(base.joinpath('dirB', 'linkD').exists( - follow_symlinks=False)) - base.joinpath('dirC')._delete() - self.assertFalse(base.joinpath('dirC').exists()) - self.assertFalse(base.joinpath('dirC', 'dirD').exists()) - self.assertFalse(base.joinpath('dirC', 'dirD', 'fileD').exists()) - self.assertFalse(base.joinpath('dirC', 'fileC').exists()) - self.assertFalse(base.joinpath('dirC', 'novel.txt').exists()) - - def test_delete_missing(self): - tmp = self.cls(self.base, 'delete') - tmp.mkdir() - # filename is guaranteed not to exist - filename = tmp / 'foo' - self.assertRaises(FileNotFoundError, filename._delete) - class DummyPathWalkTest(unittest.TestCase): cls = DummyPath From a0088b40bb212dc132e147d04f9287cabd72d163 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 4 Jan 2025 16:44:34 +0100 Subject: [PATCH 365/427] Docs: mark up json.dump() using parameter list (#128482) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/json.rst | 118 +++++++++++++++++++++++++------------------ 1 file changed, 70 insertions(+), 48 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 758d47462b6e12..f11109fb0ecfed 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -151,69 +151,91 @@ Basic Usage sort_keys=False, **kw) Serialize *obj* as a JSON formatted stream to *fp* (a ``.write()``-supporting - :term:`file-like object`) using this :ref:`conversion table + :term:`file-like object`) using this :ref:`Python-to-JSON conversion table `. - If *skipkeys* is true (default: ``False``), then dict keys that are not - of a basic type (:class:`str`, :class:`int`, :class:`float`, :class:`bool`, - ``None``) will be skipped instead of raising a :exc:`TypeError`. - - The :mod:`json` module always produces :class:`str` objects, not - :class:`bytes` objects. Therefore, ``fp.write()`` must support :class:`str` - input. - - If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. + To use a custom :class:`JSONEncoder` subclass (for example, one that overrides the + :meth:`~JSONEncoder.default` method to serialize additional types), specify it with the + *cls* keyword argument; otherwise :class:`JSONEncoder` is used. - If *check_circular* is false (default: ``True``), then the circular - reference check for container types will be skipped and a circular reference - will result in a :exc:`RecursionError` (or worse). + .. note:: - If *allow_nan* is false (default: ``True``), then it will be a - :exc:`ValueError` to serialize out of range :class:`float` values (``nan``, - ``inf``, ``-inf``) in strict compliance of the JSON specification. - If *allow_nan* is true, their JavaScript equivalents (``NaN``, - ``Infinity``, ``-Infinity``) will be used. + Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol, + so trying to serialize multiple objects with repeated calls to + :func:`dump` using the same *fp* will result in an invalid JSON file. - If *indent* is a non-negative integer or string, then JSON array elements and - object members will be pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the most compact representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. + :param object obj: + The Python object to be serialized. + + :param fp: + The file-like object *obj* will be serialized to. + The :mod:`!json` module always produces :class:`str` objects, + not :class:`bytes` objects, + therefore ``fp.write()`` must support :class:`str` input. + :type fp: :term:`file-like object` + + :param bool skipkeys: + If ``True``, keys that are not of a basic type + (:class:`str`, :class:`int`, :class:`float`, :class:`bool`, ``None``) + will be skipped instead of raising a :exc:`TypeError`. + Default ``False``. + + :param bool ensure_ascii: + If ``True`` (the default), the output is guaranteed to + have all incoming non-ASCII characters escaped. + If ``False``, these characters will be outputted as-is. + + :param bool check_circular: + If ``False``, the circular reference check for container types is skipped + and a circular reference will result in a :exc:`RecursionError` (or worse). + Default ``True``. + + :param bool allow_nan: + If ``False``, serialization of out-of-range :class:`float` values + (``nan``, ``inf``, ``-inf``) will result in a :exc:`ValueError`, + in strict compliance with the JSON specification. + If ``True`` (the default), their JavaScript equivalents + (``NaN``, ``Infinity``, ``-Infinity``) are used. + + :param indent: + If a positive integer or string, JSON array elements and + object members will be pretty-printed with that indent level. + A positive integer indents that many spaces per level; + a string (such as ``"\t"``) is used to indent each level. + If zero, negative, or ``""`` (the empty string), + only newlines are inserted. + If ``None`` (the default), the most compact representation is used. + :type indent: int | str | None + + :param separators: + A two-tuple: ``(item_separator, key_separator)``. + If ``None`` (the default), *separators* defaults to + ``(', ', ': ')`` if *indent* is ``None``, + and ``(',', ': ')`` otherwise. + For the most compact JSON, + specify ``(',', ':')`` to eliminate whitespace. + :type separators: tuple | None + + :param default: + A function that is called for objects that can't otherwise be serialized. + It should return a JSON encodable version of the object + or raise a :exc:`TypeError`. + If ``None`` (the default), :exc:`!TypeError` is raised. + :type default: :term:`callable` | None + + :param sort_keys: + If ``True``, dictionaries will be outputted sorted by key. + Default ``False``. .. versionchanged:: 3.2 Allow strings for *indent* in addition to integers. - If specified, *separators* should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. - .. versionchanged:: 3.4 Use ``(',', ': ')`` as default if *indent* is not ``None``. - If specified, *default* should be a function that gets called for objects that - can't otherwise be serialized. It should return a JSON encodable version of - the object or raise a :exc:`TypeError`. If not specified, :exc:`TypeError` - is raised. - - If *sort_keys* is true (default: ``False``), then the output of - dictionaries will be sorted by key. - - To use a custom :class:`JSONEncoder` subclass (e.g. one that overrides the - :meth:`~JSONEncoder.default` method to serialize additional types), specify it with the - *cls* kwarg; otherwise :class:`JSONEncoder` is used. - .. versionchanged:: 3.6 All optional parameters are now :ref:`keyword-only `. - .. note:: - - Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol, - so trying to serialize multiple objects with repeated calls to - :func:`dump` using the same *fp* will result in an invalid JSON file. .. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \ check_circular=True, allow_nan=True, cls=None, \ From fd94c6a8032676d0659aa9e38cdaa7c17093119c Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 4 Jan 2025 15:45:24 +0000 Subject: [PATCH 366/427] pathlib tests: create `walk()` test hierarchy without using class under test (#128338) In the tests for `pathlib.Path.walk()`, avoid using the path class under test (`self.cls`) in test setup. Instead we use `os` functions in `test_pathlib`, and direct manipulation of `DummyPath` internal data in `test_pathlib_abc`. --- Lib/test/test_pathlib/test_pathlib.py | 38 ++++++++++++++- Lib/test/test_pathlib/test_pathlib_abc.py | 59 ++++++++--------------- 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index a67a1c531630a1..6548577f4de12c 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -3029,6 +3029,42 @@ def setUp(self): if name in _tests_needing_symlinks and not self.can_symlink: self.skipTest('requires symlinks') super().setUp() + + def createTestHierarchy(self): + # Build: + # TESTFN/ + # TEST1/ a file kid and two directory kids + # tmp1 + # SUB1/ a file kid and a directory kid + # tmp2 + # SUB11/ no kids + # SUB2/ a file kid and a dirsymlink kid + # tmp3 + # link/ a symlink to TEST2 + # broken_link + # broken_link2 + # TEST2/ + # tmp4 a lone file + t2_path = self.cls(self.base, "TEST2") + os.makedirs(self.sub11_path) + os.makedirs(self.sub2_path) + os.makedirs(t2_path) + + tmp1_path = self.walk_path / "tmp1" + tmp2_path = self.sub1_path / "tmp2" + tmp3_path = self.sub2_path / "tmp3" + tmp4_path = self.cls(self.base, "TEST2", "tmp4") + for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: + with open(path, "w", encoding='utf-8') as f: + f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") + + if self.can_symlink: + broken_link_path = self.sub2_path / "broken_link" + broken_link2_path = self.sub2_path / "broken_link2" + os.symlink(t2_path, self.link_path, target_is_directory=True) + os.symlink('broken', broken_link_path) + os.symlink(os.path.join('tmp3', 'broken'), broken_link2_path) + self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) sub21_path= self.sub2_path / "SUB21" tmp5_path = sub21_path / "tmp3" broken_link3_path = self.sub2_path / "broken_link3" @@ -3052,7 +3088,7 @@ def setUp(self): def tearDown(self): if 'SUB21' in self.sub2_tree[1]: os.chmod(self.sub2_path / "SUB21", stat.S_IRWXU) - super().tearDown() + os_helper.rmtree(self.base) def test_walk_bad_dir(self): errors = [] diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 0762f224fc9ac4..87aef0c130cf9e 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1580,52 +1580,35 @@ class DummyPathWalkTest(unittest.TestCase): can_symlink = False def setUp(self): - # Build: - # TESTFN/ - # TEST1/ a file kid and two directory kids - # tmp1 - # SUB1/ a file kid and a directory kid - # tmp2 - # SUB11/ no kids - # SUB2/ a file kid and a dirsymlink kid - # tmp3 - # link/ a symlink to TEST2 - # broken_link - # broken_link2 - # TEST2/ - # tmp4 a lone file self.walk_path = self.cls(self.base, "TEST1") self.sub1_path = self.walk_path / "SUB1" self.sub11_path = self.sub1_path / "SUB11" self.sub2_path = self.walk_path / "SUB2" - tmp1_path = self.walk_path / "tmp1" - tmp2_path = self.sub1_path / "tmp2" - tmp3_path = self.sub2_path / "tmp3" self.link_path = self.sub2_path / "link" - t2_path = self.cls(self.base, "TEST2") - tmp4_path = self.cls(self.base, "TEST2", "tmp4") - broken_link_path = self.sub2_path / "broken_link" - broken_link2_path = self.sub2_path / "broken_link2" - - self.sub11_path.mkdir(parents=True) - self.sub2_path.mkdir(parents=True) - t2_path.mkdir(parents=True) - - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: - with path.open("w", encoding='utf-8') as f: - f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") + self.sub2_tree = (self.sub2_path, [], ["tmp3"]) + self.createTestHierarchy() - if self.can_symlink: - self.link_path.symlink_to(t2_path, target_is_directory=True) - broken_link_path.symlink_to('broken') - broken_link2_path.symlink_to(self.cls('tmp3', 'broken')) - self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) - else: - self.sub2_tree = (self.sub2_path, [], ["tmp3"]) + def createTestHierarchy(self): + cls = self.cls + cls._files = { + f'{self.base}/TEST1/tmp1': b'this is tmp1\n', + f'{self.base}/TEST1/SUB1/tmp2': b'this is tmp2\n', + f'{self.base}/TEST1/SUB2/tmp3': b'this is tmp3\n', + f'{self.base}/TEST2/tmp4': b'this is tmp4\n', + } + cls._directories = { + f'{self.base}': {'TEST1', 'TEST2'}, + f'{self.base}/TEST1': {'SUB1', 'SUB2', 'tmp1'}, + f'{self.base}/TEST1/SUB1': {'SUB11', 'tmp2'}, + f'{self.base}/TEST1/SUB1/SUB11': set(), + f'{self.base}/TEST1/SUB2': {'tmp3'}, + f'{self.base}/TEST2': {'tmp4'}, + } def tearDown(self): - base = self.cls(self.base) - base._delete() + cls = self.cls + cls._files.clear() + cls._directories.clear() def test_walk_topdown(self): walker = self.walk_path.walk() From f28d471fbe99f9eaac05d60ed40da47b0b56fe86 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 4 Jan 2025 17:38:57 +0000 Subject: [PATCH 367/427] gh-126719: Clarify math.fmod docs (#127741) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/math.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/math.rst b/Doc/library/math.rst index bf79b23a72bbf9..c78b313db5152d 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -248,7 +248,8 @@ Floating point arithmetic .. function:: fmod(x, y) - Return ``fmod(x, y)``, as defined by the platform C library. Note that the + Return the floating-point remainder of ``x / y``, + as defined by the platform C library function ``fmod(x, y)``. Note that the Python expression ``x % y`` may not return the same result. The intent of the C standard is that ``fmod(x, y)`` be exactly (mathematically; to infinite precision) equal to ``x - n*y`` for some integer *n* such that the result has From 87ee76062a7eb9c0fa2b94e36cfed21d86ae90ac Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 4 Jan 2025 19:57:59 +0100 Subject: [PATCH 368/427] Docs: amend json.dump() post gh-128482 (#128489) --- Doc/library/json.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index f11109fb0ecfed..169291f74f44a5 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -154,10 +154,6 @@ Basic Usage :term:`file-like object`) using this :ref:`Python-to-JSON conversion table `. - To use a custom :class:`JSONEncoder` subclass (for example, one that overrides the - :meth:`~JSONEncoder.default` method to serialize additional types), specify it with the - *cls* keyword argument; otherwise :class:`JSONEncoder` is used. - .. note:: Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol, @@ -197,6 +193,13 @@ Basic Usage If ``True`` (the default), their JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``) are used. + :param cls: + If set, a custom JSON encoder with the + :meth:`~JSONEncoder.default` method overridden, + for serializing into custom datatypes. + If ``None`` (the default), :class:`!JSONEncoder` is used. + :type cls: a :class:`JSONEncoder` subclass + :param indent: If a positive integer or string, JSON array elements and object members will be pretty-printed with that indent level. @@ -223,7 +226,7 @@ Basic Usage If ``None`` (the default), :exc:`!TypeError` is raised. :type default: :term:`callable` | None - :param sort_keys: + :param bool sort_keys: If ``True``, dictionaries will be outputted sorted by key. Default ``False``. From 8ade15343d5daec3bf79ff7c47f03726fb2bcadf Mon Sep 17 00:00:00 2001 From: "RUANG (James Roy)" Date: Sun, 5 Jan 2025 04:48:20 +0800 Subject: [PATCH 369/427] gh-127954: Document PyObject_DelItemString (#127986) --- Doc/c-api/object.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a137688fe07545..3a434a4173eafa 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -493,6 +493,13 @@ Object Protocol on failure. This is equivalent to the Python statement ``del o[key]``. +.. c:function:: int PyObject_DelItemString(PyObject *o, const char *key) + + This is the same as :c:func:`PyObject_DelItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. c:function:: PyObject* PyObject_Dir(PyObject *o) This is equivalent to the Python expression ``dir(o)``, returning a (possibly From e8b6b39ff707378da654e15ccddde9c28cefdd30 Mon Sep 17 00:00:00 2001 From: Beomsoo Kim Date: Sun, 5 Jan 2025 07:38:49 +0900 Subject: [PATCH 370/427] gh-127553: Remove outdated TODO comment in _pydatetime (#127564) --- Lib/_pydatetime.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index ed01670cfece43..be90c9b1315d53 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -2392,7 +2392,6 @@ def __reduce__(self): def _isoweek1monday(year): # Helper to calculate the day number of the Monday starting week 1 - # XXX This could be done more efficiently THURSDAY = 3 firstday = _ymd2ord(year, 1, 1) firstweekday = (firstday + 6) % 7 # See weekday() above From 0cafa97932c6574734bbaa07180bbd5a762b01a6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 5 Jan 2025 00:38:46 +0100 Subject: [PATCH 371/427] gh-115765: Document and enforce Autoconf 2.72 requirement (#128502) --- Doc/using/configure.rst | 5 ++++- configure.ac | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index e7733a6dc11451..82df8dfc948ed2 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -29,7 +29,7 @@ Features and minimum versions required to build CPython: * Tcl/Tk 8.5.12 for the :mod:`tkinter` module. -* Autoconf 2.71 and aclocal 1.16.5 are required to regenerate the +* Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the :file:`configure` script. .. versionchanged:: 3.1 @@ -58,6 +58,9 @@ Features and minimum versions required to build CPython: .. versionchanged:: 3.13 Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. +.. versionchanged:: next + Autoconf 2.72 is now required. + See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". diff --git a/configure.ac b/configure.ac index 9e131ed1a2dc98..1def41176337e3 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ dnl ************************************************************ dnl * Please run autoreconf -ivf -Werror to test your changes! * dnl ************************************************************ dnl -dnl Python's configure script requires autoconf 2.71, autoconf-archive, +dnl Python's configure script requires autoconf 2.72, autoconf-archive, dnl aclocal 1.16, and pkg-config. dnl dnl It is recommended to use the Tools/build/regen-configure.sh shell script @@ -12,7 +12,7 @@ dnl # Set VERSION so we only need to edit in one place (i.e., here) m4_define([PYTHON_VERSION], [3.14]) -AC_PREREQ([2.71]) +AC_PREREQ([2.72]) AC_INIT([python],[PYTHON_VERSION],[https://github.com/python/cpython/issues/]) From b60044b838f3ea97395cd6f3adbd5330356fc273 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Sat, 4 Jan 2025 22:16:30 -0600 Subject: [PATCH 372/427] gh-128437: Add `BOLT_COMMON_FLAGS` with `-update-debug-sections` (gh-128455) Add `BOLT_COMMON_FLAGS` with `-update-debug-sections` Co-authored-by: Gregory Szorc --- configure | 18 ++++++++++++++++-- configure.ac | 18 ++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/configure b/configure index aa88c74c61156c..61ee51c4b36473 100755 --- a/configure +++ b/configure @@ -907,6 +907,7 @@ CFLAGS_ALIASING OPT BOLT_APPLY_FLAGS BOLT_INSTRUMENT_FLAGS +BOLT_COMMON_FLAGS BOLT_BINARIES MERGE_FDATA LLVM_BOLT @@ -1142,6 +1143,7 @@ LIBS CPPFLAGS CPP PROFILE_TASK +BOLT_COMMON_FLAGS BOLT_INSTRUMENT_FLAGS BOLT_APPLY_FLAGS LIBUUID_CFLAGS @@ -1963,6 +1965,8 @@ Some influential environment variables: CPP C preprocessor PROFILE_TASK Python args for PGO generation task + BOLT_COMMON_FLAGS + Common arguments to llvm-bolt when instrumenting and applying BOLT_INSTRUMENT_FLAGS Arguments to llvm-bolt when instrumenting binaries BOLT_APPLY_FLAGS @@ -9389,11 +9393,21 @@ then : fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking BOLT_COMMON_FLAGS" >&5 +printf %s "checking BOLT_COMMON_FLAGS... " >&6; } +if test -z "${BOLT_COMMON_FLAGS}" +then + BOLT_COMMON_FLAGS=-update-debug-sections + +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking BOLT_INSTRUMENT_FLAGS" >&5 printf %s "checking BOLT_INSTRUMENT_FLAGS... " >&6; } if test -z "${BOLT_INSTRUMENT_FLAGS}" then - BOLT_INSTRUMENT_FLAGS= + BOLT_INSTRUMENT_FLAGS="${BOLT_COMMON_FLAGS}" fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BOLT_INSTRUMENT_FLAGS" >&5 printf "%s\n" "$BOLT_INSTRUMENT_FLAGS" >&6; } @@ -9403,7 +9417,7 @@ printf "%s\n" "$BOLT_INSTRUMENT_FLAGS" >&6; } printf %s "checking BOLT_APPLY_FLAGS... " >&6; } if test -z "${BOLT_APPLY_FLAGS}" then - BOLT_APPLY_FLAGS=" -update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot " + BOLT_APPLY_FLAGS=" ${BOLT_COMMON_FLAGS} -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot " fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BOLT_APPLY_FLAGS" >&5 diff --git a/configure.ac b/configure.ac index 1def41176337e3..172e8a1a842010 100644 --- a/configure.ac +++ b/configure.ac @@ -2160,6 +2160,20 @@ AS_VAR_IF([enable_shared], [yes], [ BOLT_BINARIES="${BOLT_BINARIES} \$(INSTSONAME)" ]) +AC_ARG_VAR( + [BOLT_COMMON_FLAGS], + [Common arguments to llvm-bolt when instrumenting and applying] +) + +AC_MSG_CHECKING([BOLT_COMMON_FLAGS]) +if test -z "${BOLT_COMMON_FLAGS}" +then + AS_VAR_SET( + [BOLT_COMMON_FLAGS], + [-update-debug-sections] + ) +fi + AC_ARG_VAR( [BOLT_INSTRUMENT_FLAGS], [Arguments to llvm-bolt when instrumenting binaries] @@ -2167,7 +2181,7 @@ AC_ARG_VAR( AC_MSG_CHECKING([BOLT_INSTRUMENT_FLAGS]) if test -z "${BOLT_INSTRUMENT_FLAGS}" then - BOLT_INSTRUMENT_FLAGS= + BOLT_INSTRUMENT_FLAGS="${BOLT_COMMON_FLAGS}" fi AC_MSG_RESULT([$BOLT_INSTRUMENT_FLAGS]) @@ -2181,7 +2195,7 @@ then AS_VAR_SET( [BOLT_APPLY_FLAGS], [m4_normalize(" - -update-debug-sections + ${BOLT_COMMON_FLAGS} -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions From ae23a012e6c8aadc4a588101cbe7bc86ede45627 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 5 Jan 2025 18:17:06 +0900 Subject: [PATCH 373/427] gh-128137: Update PyASCIIObject to handle interned field with the atomic operation (gh-128196) --- Include/cpython/unicodeobject.h | 20 ++++++++++++------- ...-12-24-01-40-12.gh-issue-128137.gsTwr_.rst | 2 ++ Objects/unicodeobject.c | 6 +++--- 3 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-24-01-40-12.gh-issue-128137.gsTwr_.rst diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 91799137101280..46a01c8e591709 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -109,7 +109,7 @@ typedef struct { 3: Interned, Immortal, and Static This categorization allows the runtime to determine the right cleanup mechanism at runtime shutdown. */ - unsigned int interned:2; + uint16_t interned; /* Character size: - PyUnicode_1BYTE_KIND (1): @@ -132,21 +132,23 @@ typedef struct { * all characters are in the range U+0000-U+10FFFF * at least one character is in the range U+10000-U+10FFFF */ - unsigned int kind:3; + unsigned short kind:3; /* Compact is with respect to the allocation scheme. Compact unicode objects only require one memory block while non-compact objects use one block for the PyUnicodeObject struct and another for its data buffer. */ - unsigned int compact:1; + unsigned short compact:1; /* The string only contains characters in the range U+0000-U+007F (ASCII) and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is set, use the PyASCIIObject structure. */ - unsigned int ascii:1; + unsigned short ascii:1; /* The object is statically allocated. */ - unsigned int statically_allocated:1; + unsigned short statically_allocated:1; /* Padding to ensure that PyUnicode_DATA() is always aligned to - 4 bytes (see issue #19537 on m68k). */ - unsigned int :24; + 4 bytes (see issue #19537 on m68k) and we use unsigned short to avoid + the extra four bytes on 32-bit Windows. This is restricted features + for specific compilers including GCC, MSVC, Clang and IBM's XL compiler. */ + unsigned short :10; } state; } PyASCIIObject; @@ -195,7 +197,11 @@ typedef struct { /* Use only if you know it's a string */ static inline unsigned int PyUnicode_CHECK_INTERNED(PyObject *op) { +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_uint16_relaxed(&_PyASCIIObject_CAST(op)->state.interned); +#else return _PyASCIIObject_CAST(op)->state.interned; +#endif } #define PyUnicode_CHECK_INTERNED(op) PyUnicode_CHECK_INTERNED(_PyObject_CAST(op)) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-24-01-40-12.gh-issue-128137.gsTwr_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-24-01-40-12.gh-issue-128137.gsTwr_.rst new file mode 100644 index 00000000000000..a3b7cde7f67676 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-24-01-40-12.gh-issue-128137.gsTwr_.rst @@ -0,0 +1,2 @@ +Update :c:type:`PyASCIIObject` layout to handle interned field with the +atomic operation. Patch by Donghee Na. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 5e532ce0f348c4..3eafa2381c1a4d 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15729,7 +15729,7 @@ immortalize_interned(PyObject *s) _Py_DecRefTotal(_PyThreadState_GET()); } #endif - _PyUnicode_STATE(s).interned = SSTATE_INTERNED_IMMORTAL; + FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL); _Py_SetImmortal(s); } @@ -15848,7 +15848,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, _Py_DecRefTotal(_PyThreadState_GET()); #endif } - _PyUnicode_STATE(s).interned = SSTATE_INTERNED_MORTAL; + FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_MORTAL); /* INTERNED_MORTAL -> INTERNED_IMMORTAL (if needed) */ @@ -15984,7 +15984,7 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) Py_UNREACHABLE(); } if (!shared) { - _PyUnicode_STATE(s).interned = SSTATE_NOT_INTERNED; + FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_NOT_INTERNED); } } #ifdef INTERNED_STATS From 2228e92da31ca7344a163498f848973a1b356597 Mon Sep 17 00:00:00 2001 From: Damien <81557462+Damien-Chen@users.noreply.github.com> Date: Sun, 5 Jan 2025 20:07:18 +0800 Subject: [PATCH 374/427] gh-128504: Upgrade doctest to ubuntu-24.04 (#128506) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/reusable-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 3962d12403919a..88da55bf08b8fe 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -99,7 +99,7 @@ jobs: # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: name: 'Doctest' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 From 3b231be8f000ae59faa04d5a2f1af11beafee866 Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Mon, 6 Jan 2025 06:58:31 +0900 Subject: [PATCH 375/427] Docs: fix `MessageDefect` references in email.policy docs (#128468) --- Doc/library/email.policy.rst | 4 ++-- Doc/tools/.nitignore | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 314767d0802a08..6b997ee784f6e4 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -267,7 +267,7 @@ added matters. To illustrate:: Handle a *defect* found on *obj*. When the email package calls this method, *defect* will always be a subclass of - :class:`~email.errors.Defect`. + :class:`~email.errors.MessageDefect`. The default implementation checks the :attr:`raise_on_defect` flag. If it is ``True``, *defect* is raised as an exception. If it is ``False`` @@ -277,7 +277,7 @@ added matters. To illustrate:: .. method:: register_defect(obj, defect) Register a *defect* on *obj*. In the email package, *defect* will always - be a subclass of :class:`~email.errors.Defect`. + be a subclass of :class:`~email.errors.MessageDefect`. The default implementation calls the ``append`` method of the ``defects`` attribute of *obj*. When the email package calls :attr:`handle_defect`, diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 7d50aec56a9bf7..6940c95ab2c9a1 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -23,7 +23,6 @@ Doc/library/email.charset.rst Doc/library/email.compat32-message.rst Doc/library/email.errors.rst Doc/library/email.parser.rst -Doc/library/email.policy.rst Doc/library/exceptions.rst Doc/library/functools.rst Doc/library/http.cookiejar.rst From a62ba52f1439c1f878a3ff9b8544caf9aeef9b90 Mon Sep 17 00:00:00 2001 From: RanKKI Date: Mon, 6 Jan 2025 12:32:16 +1100 Subject: [PATCH 376/427] gh-98188: Fix EmailMessage.get_payload to decode data when CTE value has extra text (#127547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Up to this point message handling has been very strict with regards to content encoding values: mixed case was accepted, but trailing blanks or other text would cause decoding failure, even if the first token was a valid encoding. By Postel's Rule we should go ahead and decode as long as we can recognize that first token. We have not thought of any security or backward compatibility concerns with this fix. This fix does introduce a new technique/pattern to the Message code: we look to see if the header has a 'cte' attribute, and if so we use that. This effectively promotes the header API exposed by HeaderRegistry to an API that any header parser "should" support. This seems like a reasonable thing to do. It is not, however, a requirement, as the string value of the header is still used if there is no cte attribute. The full fix (ignore any trailing blanks or blank-separated trailing text) applies only to the non-compat32 API. compat32 is only fixed to the extent that it now ignores trailing spaces. Note that the HeaderRegistry parsing still records a HeaderDefect if there is extra text. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/email/message.py | 8 +++- Lib/test/test_email/test_email.py | 44 +++++++++++++++++++ Lib/test/test_email/test_headerregistry.py | 5 +++ Misc/ACKS | 1 + ...4-12-03-14-45-16.gh-issue-98188.GX9i2b.rst | 3 ++ 5 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-03-14-45-16.gh-issue-98188.GX9i2b.rst diff --git a/Lib/email/message.py b/Lib/email/message.py index a58afc5fe5f68e..87fcab68868d46 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -286,8 +286,12 @@ def get_payload(self, i=None, decode=False): if i is not None and not isinstance(self._payload, list): raise TypeError('Expected list, got %s' % type(self._payload)) payload = self._payload - # cte might be a Header, so for now stringify it. - cte = str(self.get('content-transfer-encoding', '')).lower() + cte = self.get('content-transfer-encoding', '') + if hasattr(cte, 'cte'): + cte = cte.cte + else: + # cte might be a Header, so for now stringify it. + cte = str(cte).strip().lower() # payload may be bytes here. if not decode: if isinstance(payload, str) and utils._has_surrogates(payload): diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index abe9ef2e94409f..2deb35721576b8 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -810,6 +810,16 @@ def test_unicode_body_defaults_to_utf8_encoding(self): w4kgdGVzdGFiYwo= """)) + def test_string_payload_with_base64_cte(self): + msg = email.message_from_string(textwrap.dedent("""\ + Content-Transfer-Encoding: base64 + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.default) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + self.assertDefectsEqual(msg['content-transfer-encoding'].defects, []) + + # Test the email.encoders module class TestEncoders(unittest.TestCase): @@ -2352,6 +2362,40 @@ def test_missing_header_body_separator(self): self.assertDefectsEqual(msg.defects, [errors.MissingHeaderBodySeparatorDefect]) + def test_string_payload_with_extra_space_after_cte(self): + # https://github.com/python/cpython/issues/98188 + cte = "base64 " + msg = email.message_from_string(textwrap.dedent(f"""\ + Content-Transfer-Encoding: {cte} + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.default) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + self.assertDefectsEqual(msg['content-transfer-encoding'].defects, []) + + def test_string_payload_with_extra_text_after_cte(self): + msg = email.message_from_string(textwrap.dedent("""\ + Content-Transfer-Encoding: base64 some text + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.default) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + cte = msg['content-transfer-encoding'] + self.assertDefectsEqual(cte.defects, [email.errors.InvalidHeaderDefect]) + + def test_string_payload_with_extra_space_after_cte_compat32(self): + cte = "base64 " + msg = email.message_from_string(textwrap.dedent(f"""\ + Content-Transfer-Encoding: {cte} + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.compat32) + pasted_cte = msg['content-transfer-encoding'] + self.assertEqual(pasted_cte, cte) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + self.assertDefectsEqual(msg.defects, []) + + # Test RFC 2047 header encoding and decoding class TestRFC2047(TestEmailBase): diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index 4c0523f410332f..ff7a6da644d572 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -837,6 +837,11 @@ def cte_as_value(self, '7bit', [errors.InvalidHeaderDefect]), + 'extra_space_after_cte': ( + 'base64 ', + 'base64', + []), + } diff --git a/Misc/ACKS b/Misc/ACKS index c6e53317b37d78..d7585c16c8169c 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1129,6 +1129,7 @@ Gregor Lingl Everett Lipman Mirko Liss Alexander Liu +Hui Liu Yuan Liu Nick Lockwood Stephanie Lockwood diff --git a/Misc/NEWS.d/next/Library/2024-12-03-14-45-16.gh-issue-98188.GX9i2b.rst b/Misc/NEWS.d/next/Library/2024-12-03-14-45-16.gh-issue-98188.GX9i2b.rst new file mode 100644 index 00000000000000..30ab8cfc3f0bc6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-03-14-45-16.gh-issue-98188.GX9i2b.rst @@ -0,0 +1,3 @@ +Fix an issue in :meth:`email.message.Message.get_payload` where data +cannot be decoded if the Content Transfer Encoding mechanism contains +trailing whitespaces or additional junk text. Patch by Hui Liu. From aef52ca8b334ff90e8032da39f4d06e7b5130eb9 Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Mon, 6 Jan 2025 09:42:26 +0100 Subject: [PATCH 377/427] gh-128519: Align the docstring of untokenize() to match the docs (#128521) --- Lib/tokenize.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 7ece4e9b70d31b..1a60fd32a77ea4 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -318,16 +318,10 @@ def untokenize(iterable): with at least two elements, a token number and token value. If only two tokens are passed, the resulting output is poor. - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited input: - # Output bytes will tokenize back to the input - t1 = [tok[:2] for tok in tokenize(f.readline)] - newcode = untokenize(t1) - readline = BytesIO(newcode).readline - t2 = [tok[:2] for tok in tokenize(readline)] - assert t1 == t2 + The result is guaranteed to tokenize back to match the input so + that the conversion is lossless and round-trips are assured. + The guarantee applies only to the token type and token string as + the spacing between tokens (column positions) may change. """ ut = Untokenizer() out = ut.untokenize(iterable) From 879d287f49edcd4fa68324265f8ba63758716540 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 6 Jan 2025 13:29:18 +0300 Subject: [PATCH 378/427] gh-102471: convert decimal module to use PyLong_Export API (PEP 757) (#128267) --- Modules/_decimal/_decimal.c | 51 ++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index c564813036e504..0def463c7d8b9e 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -30,7 +30,6 @@ #endif #include -#include "pycore_long.h" // _PyLong_IsZero() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_typeobject.h" #include "complexobject.h" @@ -2323,38 +2322,42 @@ static PyObject * dec_from_long(decimal_state *state, PyTypeObject *type, PyObject *v, const mpd_context_t *ctx, uint32_t *status) { - PyObject *dec; - PyLongObject *l = (PyLongObject *)v; + PyObject *dec = PyDecType_New(state, type); - dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } - if (_PyLong_IsZero(l)) { - _dec_settriple(dec, MPD_POS, 0, 0); - return dec; - } - - uint8_t sign = _PyLong_IsNegative(l) ? MPD_NEG : MPD_POS; + PyLongExport export_long; - if (_PyLong_IsCompact(l)) { - _dec_settriple(dec, sign, l->long_value.ob_digit[0], 0); - mpd_qfinalize(MPD(dec), ctx, status); - return dec; + if (PyLong_Export(v, &export_long) == -1) { + Py_DECREF(dec); + return NULL; } - size_t len = _PyLong_DigitCount(l); + if (export_long.digits) { + const PyLongLayout *layout = PyLong_GetNativeLayout(); + uint32_t base = (uint32_t)1 << layout->bits_per_digit; + uint8_t sign = export_long.negative ? MPD_NEG : MPD_POS; + Py_ssize_t len = export_long.ndigits; -#if PYLONG_BITS_IN_DIGIT == 30 - mpd_qimport_u32(MPD(dec), l->long_value.ob_digit, len, sign, PyLong_BASE, - ctx, status); -#elif PYLONG_BITS_IN_DIGIT == 15 - mpd_qimport_u16(MPD(dec), l->long_value.ob_digit, len, sign, PyLong_BASE, - ctx, status); -#else - #error "PYLONG_BITS_IN_DIGIT should be 15 or 30" -#endif + assert(layout->bits_per_digit <= 32); + assert(layout->digits_order == -1); + assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1)); + assert(layout->digit_size == 2 || layout->digit_size == 4); + if (layout->digit_size == 4) { + mpd_qimport_u32(MPD(dec), export_long.digits, len, sign, + base, ctx, status); + } + else { + mpd_qimport_u16(MPD(dec), export_long.digits, len, sign, + base, ctx, status); + } + PyLong_FreeExport(&export_long); + } + else { + mpd_qset_i64(MPD(dec), export_long.value, ctx, status); + } return dec; } From d50fa0541b6d4f458d7ab16d3a11b1117c607f85 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 6 Jan 2025 02:30:42 -0800 Subject: [PATCH 379/427] gh-128089: Add PYC magic number for VALUE_WITH_FAKE_GLOBALS (#128097) Assign 3610 PYC magic number to VALUE_WITH_FAKE_GLOBALS format of annotationlib. --- Include/internal/pycore_magic_number.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 4c3b9c4c71da1b..ec3685d2034560 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -262,7 +262,7 @@ Known values: Python 3.14a1 3607 (Add pseudo instructions JUMP_IF_TRUE/FALSE) Python 3.14a1 3608 (Add support for slices) Python 3.14a2 3609 (Add LOAD_SMALL_INT and LOAD_CONST_IMMORTAL instructions, remove RETURN_CONST) - (3610 accidentally omitted) + Python 3.14a4 3610 (Add VALUE_WITH_FAKE_GLOBALS format to annotationlib) Python 3.14a4 3611 (Add NOT_TAKEN instruction) Python 3.15 will start with 3650 From 8d15058d61681e98579cf3fbd3c6fd13b3df6a72 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 6 Jan 2025 12:50:28 +0200 Subject: [PATCH 380/427] gh-77214: Update outdated documentation for numeric PyArg_Parse formats (GH-128454) --- Doc/c-api/arg.rst | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 3201bdc82691f4..41c0366d205086 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -229,12 +229,24 @@ There are three ways strings and buffers can be converted to C: Numbers ------- +These formats allow representing Python numbers or single characters as C numbers. +Formats that require :class:`int`, :class:`float` or :class:`complex` can +also use the corresponding special methods :meth:`~object.__index__`, +:meth:`~object.__float__` or :meth:`~object.__complex__` to convert +the Python object to the required type. + +For signed integer formats, :exc:`OverflowError` is raised if the value +is out of range for the C type. +For unsigned integer formats, no range checking is done --- the +most significant bits are silently truncated when the receiving field is too +small to receive the value. + ``b`` (:class:`int`) [unsigned char] - Convert a nonnegative Python integer to an unsigned tiny int, stored in a C + Convert a nonnegative Python integer to an unsigned tiny integer, stored in a C :c:expr:`unsigned char`. ``B`` (:class:`int`) [unsigned char] - Convert a Python integer to a tiny int without overflow checking, stored in a C + Convert a Python integer to a tiny integer without overflow checking, stored in a C :c:expr:`unsigned char`. ``h`` (:class:`int`) [short int] @@ -344,12 +356,6 @@ Other objects in *items*. The C arguments must correspond to the individual format units in *items*. Format units for sequences may be nested. -It is possible to pass "long" integers (integers whose value exceeds the -platform's :c:macro:`LONG_MAX`) however no proper range checking is done --- the -most significant bits are silently truncated when the receiving field is too -small to receive the value (actually, the semantics are inherited from downcasts -in C --- your mileage may vary). - A few other characters have a meaning in a format string. These may not occur inside nested parentheses. They are: From 1ef6bf4e29db43bbf06639923516838db2d5a5ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 6 Jan 2025 12:50:01 +0100 Subject: [PATCH 381/427] gh-111178: fix UBSan failures in `Objects/descrobject.c` (GH-128245) fix UBSan failures for `propertyobject` --- Objects/descrobject.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 4eccd1704eb95a..1852118359f014 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1508,6 +1508,8 @@ PyWrapper_New(PyObject *d, PyObject *self) /* A built-in 'property' type */ +#define _propertyobject_CAST(op) ((propertyobject *)(op)) + /* class property(object): @@ -1911,8 +1913,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, } static PyObject * -property_get__name__(propertyobject *prop, void *Py_UNUSED(ignored)) +property_get__name__(PyObject *op, void *Py_UNUSED(ignored)) { + propertyobject *prop = _propertyobject_CAST(op); PyObject *name; if (property_name(prop, &name) < 0) { return NULL; @@ -1925,16 +1928,17 @@ property_get__name__(propertyobject *prop, void *Py_UNUSED(ignored)) } static int -property_set__name__(propertyobject *prop, PyObject *value, - void *Py_UNUSED(ignored)) +property_set__name__(PyObject *op, PyObject *value, void *Py_UNUSED(ignored)) { + propertyobject *prop = _propertyobject_CAST(op); Py_XSETREF(prop->prop_name, Py_XNewRef(value)); return 0; } static PyObject * -property_get___isabstractmethod__(propertyobject *prop, void *closure) +property_get___isabstractmethod__(PyObject *op, void *closure) { + propertyobject *prop = _propertyobject_CAST(op); int res = _PyObject_IsAbstract(prop->prop_get); if (res == -1) { return NULL; @@ -1962,9 +1966,8 @@ property_get___isabstractmethod__(propertyobject *prop, void *closure) } static PyGetSetDef property_getsetlist[] = { - {"__name__", (getter)property_get__name__, (setter)property_set__name__}, - {"__isabstractmethod__", - (getter)property_get___isabstractmethod__, NULL, + {"__name__", property_get__name__, property_set__name__, NULL, NULL}, + {"__isabstractmethod__", property_get___isabstractmethod__, NULL, NULL, NULL}, {NULL} /* Sentinel */ From 657d7b77e5c69967e9c0000b986fa4872d13958c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 6 Jan 2025 14:28:50 +0200 Subject: [PATCH 382/427] gh-90241: Clarify documentation for PyUnicode_FSConverter and PyUnicode_FSDecoder (GH-128451) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Erlend E. Aasland --- Doc/c-api/arg.rst | 12 +++++++++--- Doc/c-api/unicode.rst | 35 +++++++++++++++++++++++++++-------- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 41c0366d205086..209056ef2f8bce 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -319,7 +319,7 @@ Other objects .. _o_ampersand: -``O&`` (object) [*converter*, *anything*] +``O&`` (object) [*converter*, *address*] Convert a Python object to a C variable through a *converter* function. This takes two arguments: the first is a function, the second is the address of a C variable (of arbitrary type), converted to :c:expr:`void *`. The *converter* @@ -333,14 +333,20 @@ Other objects the conversion has failed. When the conversion fails, the *converter* function should raise an exception and leave the content of *address* unmodified. - If the *converter* returns ``Py_CLEANUP_SUPPORTED``, it may get called a + .. c:macro:: Py_CLEANUP_SUPPORTED + :no-typesetting: + + If the *converter* returns :c:macro:`!Py_CLEANUP_SUPPORTED`, it may get called a second time if the argument parsing eventually fails, giving the converter a chance to release any memory that it had already allocated. In this second call, the *object* parameter will be ``NULL``; *address* will have the same value as in the original call. + Examples of converters: :c:func:`PyUnicode_FSConverter` and + :c:func:`PyUnicode_FSDecoder`. + .. versionchanged:: 3.1 - ``Py_CLEANUP_SUPPORTED`` was added. + :c:macro:`!Py_CLEANUP_SUPPORTED` was added. ``p`` (:class:`bool`) [int] Tests the value passed in for truth (a boolean **p**\ redicate) and converts diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index dcbc8804cd6b89..f19b86a8dbfb66 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -786,16 +786,25 @@ Functions encoding to and decoding from the :term:`filesystem encoding and error handler` (:pep:`383` and :pep:`529`). To encode file names to :class:`bytes` during argument parsing, the ``"O&"`` -converter should be used, passing :c:func:`PyUnicode_FSConverter` as the +converter should be used, passing :c:func:`!PyUnicode_FSConverter` as the conversion function: .. c:function:: int PyUnicode_FSConverter(PyObject* obj, void* result) - ParseTuple converter: encode :class:`str` objects -- obtained directly or + :ref:`PyArg_Parse\* converter `: encode :class:`str` objects -- obtained directly or through the :class:`os.PathLike` interface -- to :class:`bytes` using :c:func:`PyUnicode_EncodeFSDefault`; :class:`bytes` objects are output as-is. - *result* must be a :c:expr:`PyBytesObject*` which must be released when it is - no longer used. + *result* must be an address of a C variable of type :c:expr:`PyObject*` + (or :c:expr:`PyBytesObject*`). + On success, set the variable to a new :term:`strong reference` to + a :ref:`bytes object ` which must be released + when it is no longer used and return a non-zero value + (:c:macro:`Py_CLEANUP_SUPPORTED`). + Embedded null bytes are not allowed in the result. + On failure, return ``0`` with an exception set. + + If *obj* is ``NULL``, the function releases a strong reference + stored in the variable referred by *result* and returns ``1``. .. versionadded:: 3.1 @@ -803,16 +812,26 @@ conversion function: Accepts a :term:`path-like object`. To decode file names to :class:`str` during argument parsing, the ``"O&"`` -converter should be used, passing :c:func:`PyUnicode_FSDecoder` as the +converter should be used, passing :c:func:`!PyUnicode_FSDecoder` as the conversion function: .. c:function:: int PyUnicode_FSDecoder(PyObject* obj, void* result) - ParseTuple converter: decode :class:`bytes` objects -- obtained either + :ref:`PyArg_Parse\* converter `: decode :class:`bytes` objects -- obtained either directly or indirectly through the :class:`os.PathLike` interface -- to :class:`str` using :c:func:`PyUnicode_DecodeFSDefaultAndSize`; :class:`str` - objects are output as-is. *result* must be a :c:expr:`PyUnicodeObject*` which - must be released when it is no longer used. + objects are output as-is. + *result* must be an address of a C variable of type :c:expr:`PyObject*` + (or :c:expr:`PyUnicodeObject*`). + On success, set the variable to a new :term:`strong reference` to + a :ref:`Unicode object ` which must be released + when it is no longer used and return a non-zero value + (:c:macro:`Py_CLEANUP_SUPPORTED`). + Embedded null characters are not allowed in the result. + On failure, return ``0`` with an exception set. + + If *obj* is ``NULL``, release the strong reference + to the object referred to by *result* and return ``1``. .. versionadded:: 3.2 From 7e8c571604cd18e65cefd26bfc48082840264549 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Mon, 6 Jan 2025 18:05:11 +0530 Subject: [PATCH 383/427] gh-128340: add thread safe handle for `loop.call_soon_threadsafe` (#128369) Adds `_ThreadSafeHandle` to be used for callbacks scheduled with `loop.call_soon_threadsafe`. --- Lib/asyncio/base_events.py | 5 +- Lib/asyncio/events.py | 28 +++++ Lib/test/test_asyncio/test_events.py | 118 ++++++++++++++++++ ...-01-05-11-46-14.gh-issue-128340.gKI0uU.rst | 1 + 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-05-11-46-14.gh-issue-128340.gKI0uU.rst diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 5dbe4b28d236d3..9e6f6e3ee7e3ec 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -873,7 +873,10 @@ def call_soon_threadsafe(self, callback, *args, context=None): self._check_closed() if self._debug: self._check_callback(callback, 'call_soon_threadsafe') - handle = self._call_soon(callback, args, context) + handle = events._ThreadSafeHandle(callback, args, self, context) + self._ready.append(handle) + if handle._source_traceback: + del handle._source_traceback[-1] if handle._source_traceback: del handle._source_traceback[-1] self._write_to_self() diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 6e291d28ec81ae..2ee9870e80f20b 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -113,6 +113,34 @@ def _run(self): self._loop.call_exception_handler(context) self = None # Needed to break cycles when an exception occurs. +# _ThreadSafeHandle is used for callbacks scheduled with call_soon_threadsafe +# and is thread safe unlike Handle which is not thread safe. +class _ThreadSafeHandle(Handle): + + __slots__ = ('_lock',) + + def __init__(self, callback, args, loop, context=None): + super().__init__(callback, args, loop, context) + self._lock = threading.RLock() + + def cancel(self): + with self._lock: + return super().cancel() + + def cancelled(self): + with self._lock: + return super().cancelled() + + def _run(self): + # The event loop checks for cancellation without holding the lock + # It is possible that the handle is cancelled after the check + # but before the callback is called so check it again after acquiring + # the lock and return without calling the callback if it is cancelled. + with self._lock: + if self._cancelled: + return + return super()._run() + class TimerHandle(Handle): """Object returned by timed callback registration methods.""" diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index c8439c9af5e6ba..ed75b909317357 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -353,6 +353,124 @@ def run_in_thread(): t.join() self.assertEqual(results, ['hello', 'world']) + def test_call_soon_threadsafe_handle_block_check_cancelled(self): + results = [] + + callback_started = threading.Event() + callback_finished = threading.Event() + def callback(arg): + callback_started.set() + results.append(arg) + time.sleep(1) + callback_finished.set() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback started so it should block checking for cancellation + # until it finishes + self.assertFalse(handle.cancelled()) + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_block_cancellation(self): + results = [] + + callback_started = threading.Event() + callback_finished = threading.Event() + def callback(arg): + callback_started.set() + results.append(arg) + time.sleep(1) + callback_finished.set() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback started so it cannot be cancelled from other thread until + # it finishes + handle.cancel() + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_cancel_same_thread(self): + results = [] + callback_started = threading.Event() + callback_finished = threading.Event() + + fut = concurrent.futures.Future() + def callback(arg): + callback_started.set() + handle = fut.result() + handle.cancel() + results.append(arg) + callback_finished.set() + self.loop.stop() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + fut.set_result(handle) + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback cancels itself from same thread so it has no effect + # it runs to completion + self.assertTrue(handle.cancelled()) + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_cancel_other_thread(self): + results = [] + ev = threading.Event() + + callback_finished = threading.Event() + def callback(arg): + results.append(arg) + callback_finished.set() + self.loop.stop() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + # handle can be cancelled from other thread if not started yet + self.assertIsInstance(handle, events._ThreadSafeHandle) + handle.cancel() + self.assertTrue(handle.cancelled()) + self.assertFalse(callback_finished.is_set()) + ev.set() + self.loop.call_soon_threadsafe(self.loop.stop) + + # block the main loop until the callback is added and cancelled in the + # other thread + self.loop.call_soon(ev.wait) + t = threading.Thread(target=run_in_thread) + t.start() + self.loop.run_forever() + t.join() + self.assertEqual(results, []) + self.assertFalse(callback_finished.is_set()) + def test_call_soon_threadsafe_same_thread(self): results = [] diff --git a/Misc/NEWS.d/next/Library/2025-01-05-11-46-14.gh-issue-128340.gKI0uU.rst b/Misc/NEWS.d/next/Library/2025-01-05-11-46-14.gh-issue-128340.gKI0uU.rst new file mode 100644 index 00000000000000..790400a19f334b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-05-11-46-14.gh-issue-128340.gKI0uU.rst @@ -0,0 +1 @@ +Add internal thread safe handle to be used in :meth:`asyncio.loop.call_soon_threadsafe` for thread safe cancellation. From f89e5e20cb8964653ea7d6f53d3e40953b6548ce Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 6 Jan 2025 13:43:09 +0100 Subject: [PATCH 384/427] gh-127350: Add Py_fopen() and Py_fclose() functions (#127821) --- Doc/c-api/sys.rst | 32 ++++++++ Doc/whatsnew/3.14.rst | 6 ++ Include/cpython/fileutils.h | 10 ++- Lib/test/test_capi/test_file.py | 67 ++++++++++++++++ Lib/test/test_ssl.py | 3 +- ...-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst | 5 ++ Modules/_ssl.c | 2 +- Modules/_ssl/debughelpers.c | 4 +- Modules/_testcapi/clinic/file.c.h | 48 +++++++++++ Modules/_testcapi/file.c | 35 ++++++++ Modules/_testcapi/object.c | 8 +- Modules/_testcapimodule.c | 12 +-- Modules/main.c | 4 +- Python/errors.c | 2 +- Python/fileutils.c | 79 ++++++++++++------- Python/import.c | 2 +- Python/pythonrun.c | 2 +- Python/sysmodule.c | 2 +- 18 files changed, 270 insertions(+), 53 deletions(-) create mode 100644 Lib/test/test_capi/test_file.py create mode 100644 Misc/NEWS.d/next/C_API/2024-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst create mode 100644 Modules/_testcapi/clinic/file.c.h diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index c688afdca8231d..7a7d39aea20baf 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -216,6 +216,38 @@ Operating System Utilities The function now uses the UTF-8 encoding on Windows if :c:member:`PyPreConfig.legacy_windows_fs_encoding` is zero. +.. c:function:: FILE* Py_fopen(PyObject *path, const char *mode) + + Similar to :c:func:`!fopen`, but *path* is a Python object and + an exception is set on error. + + *path* must be a :class:`str` object, a :class:`bytes` object, + or a :term:`path-like object`. + + On success, return the new file pointer. + On error, set an exception and return ``NULL``. + + The file must be closed by :c:func:`Py_fclose` rather than calling directly + :c:func:`!fclose`. + + The file descriptor is created non-inheritable (:pep:`446`). + + The caller must hold the GIL. + + .. versionadded:: next + + +.. c:function:: int Py_fclose(FILE *file) + + Close a file that was opened by :c:func:`Py_fopen`. + + On success, return ``0``. + On error, return ``EOF`` and ``errno`` is set to indicate the error. + In either case, any further access (including another call to + :c:func:`Py_fclose`) to the stream results in undefined behavior. + + .. versionadded:: next + .. _systemfunctions: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index f365db37217e95..16851b4e63ea2c 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1237,6 +1237,12 @@ New features :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` events, respectively. +* Add :c:func:`Py_fopen` function to open a file. Similar to the + :c:func:`!fopen` function, but the *path* parameter is a Python object and an + exception is set on error. Add also :c:func:`Py_fclose` function to close a + file. + (Contributed by Victor Stinner in :gh:`127350`.) + Porting to Python 3.14 ---------------------- diff --git a/Include/cpython/fileutils.h b/Include/cpython/fileutils.h index b386ad107bde1f..702f89aca324c5 100644 --- a/Include/cpython/fileutils.h +++ b/Include/cpython/fileutils.h @@ -2,7 +2,13 @@ # error "this header file must not be included directly" #endif -// Used by _testcapi which must not use the internal C API -PyAPI_FUNC(FILE*) _Py_fopen_obj( +PyAPI_FUNC(FILE*) Py_fopen( PyObject *path, const char *mode); + +// Deprecated alias to Py_fopen() kept for backward compatibility +Py_DEPRECATED(3.14) PyAPI_FUNC(FILE*) _Py_fopen_obj( + PyObject *path, + const char *mode); + +PyAPI_FUNC(int) Py_fclose(FILE *file); diff --git a/Lib/test/test_capi/test_file.py b/Lib/test/test_capi/test_file.py new file mode 100644 index 00000000000000..8a08a0a93eb8b7 --- /dev/null +++ b/Lib/test/test_capi/test_file.py @@ -0,0 +1,67 @@ +import os +import unittest +from test import support +from test.support import import_helper, os_helper + +_testcapi = import_helper.import_module('_testcapi') + + +class CAPIFileTest(unittest.TestCase): + def test_py_fopen(self): + # Test Py_fopen() and Py_fclose() + + with open(__file__, "rb") as fp: + source = fp.read() + + for filename in (__file__, os.fsencode(__file__)): + with self.subTest(filename=filename): + data = _testcapi.py_fopen(filename, "rb") + self.assertEqual(data, source[:256]) + + data = _testcapi.py_fopen(os_helper.FakePath(filename), "rb") + self.assertEqual(data, source[:256]) + + filenames = [ + os_helper.TESTFN, + os.fsencode(os_helper.TESTFN), + ] + # TESTFN_UNDECODABLE cannot be used to create a file on macOS/WASI. + if os_helper.TESTFN_UNENCODABLE is not None: + filenames.append(os_helper.TESTFN_UNENCODABLE) + for filename in filenames: + with self.subTest(filename=filename): + try: + with open(filename, "wb") as fp: + fp.write(source) + + data = _testcapi.py_fopen(filename, "rb") + self.assertEqual(data, source[:256]) + finally: + os_helper.unlink(filename) + + # embedded null character/byte in the filename + with self.assertRaises(ValueError): + _testcapi.py_fopen("a\x00b", "rb") + with self.assertRaises(ValueError): + _testcapi.py_fopen(b"a\x00b", "rb") + + # non-ASCII mode failing with "Invalid argument" + with self.assertRaises(OSError): + _testcapi.py_fopen(__file__, "\xe9") + + # invalid filename type + for invalid_type in (123, object()): + with self.subTest(filename=invalid_type): + with self.assertRaises(TypeError): + _testcapi.py_fopen(invalid_type, "rb") + + if support.MS_WINDOWS: + with self.assertRaises(OSError): + # On Windows, the file mode is limited to 10 characters + _testcapi.py_fopen(__file__, "rt+, ccs=UTF-8") + + # CRASHES py_fopen(__file__, None) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index c16ef3f96f9a21..9863f3ffe97656 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1325,8 +1325,7 @@ def test_load_verify_cadata(self): def test_load_dh_params(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.load_dh_params(DHFILE) - if os.name != 'nt': - ctx.load_dh_params(BYTES_DHFILE) + ctx.load_dh_params(BYTES_DHFILE) self.assertRaises(TypeError, ctx.load_dh_params) self.assertRaises(TypeError, ctx.load_dh_params, None) with self.assertRaises(FileNotFoundError) as cm: diff --git a/Misc/NEWS.d/next/C_API/2024-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst b/Misc/NEWS.d/next/C_API/2024-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst new file mode 100644 index 00000000000000..d1b528c673442f --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst @@ -0,0 +1,5 @@ +Add :c:func:`Py_fopen` function to open a file. Similar to the :c:func:`!fopen` +function, but the *path* parameter is a Python object and an exception is set +on error. Add also :c:func:`Py_fclose` function to close a file, function +needed for Windows support. +Patch by Victor Stinner. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 74cf99957389e2..87739832fbf784 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4377,7 +4377,7 @@ _ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath) FILE *f; DH *dh; - f = _Py_fopen_obj(filepath, "rb"); + f = Py_fopen(filepath, "rb"); if (f == NULL) return NULL; diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 9c87f8b4d21e68..318c045a0eec3c 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -180,8 +180,8 @@ _PySSLContext_set_keylog_filename(PySSLContext *self, PyObject *arg, void *c) { return 0; } - /* _Py_fopen_obj() also checks that arg is of proper type. */ - fp = _Py_fopen_obj(arg, "a" PY_STDIOTEXTMODE); + /* Py_fopen() also checks that arg is of proper type. */ + fp = Py_fopen(arg, "a" PY_STDIOTEXTMODE); if (fp == NULL) return -1; diff --git a/Modules/_testcapi/clinic/file.c.h b/Modules/_testcapi/clinic/file.c.h new file mode 100644 index 00000000000000..2ca21fffcef680 --- /dev/null +++ b/Modules/_testcapi/clinic/file.c.h @@ -0,0 +1,48 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(_testcapi_py_fopen__doc__, +"py_fopen($module, path, mode, /)\n" +"--\n" +"\n" +"Call Py_fopen(), fread(256) and Py_fclose(). Return read bytes."); + +#define _TESTCAPI_PY_FOPEN_METHODDEF \ + {"py_fopen", _PyCFunction_CAST(_testcapi_py_fopen), METH_FASTCALL, _testcapi_py_fopen__doc__}, + +static PyObject * +_testcapi_py_fopen_impl(PyObject *module, PyObject *path, const char *mode); + +static PyObject * +_testcapi_py_fopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *path; + const char *mode; + + if (!_PyArg_CheckPositional("py_fopen", nargs, 2, 2)) { + goto exit; + } + path = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("py_fopen", "argument 2", "str", args[1]); + goto exit; + } + Py_ssize_t mode_length; + mode = PyUnicode_AsUTF8AndSize(args[1], &mode_length); + if (mode == NULL) { + goto exit; + } + if (strlen(mode) != (size_t)mode_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = _testcapi_py_fopen_impl(module, path, mode); + +exit: + return return_value; +} +/*[clinic end generated code: output=c9fe964c3e5a0c32 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/file.c b/Modules/_testcapi/file.c index 634563f6ea12cb..4bad43010fd440 100644 --- a/Modules/_testcapi/file.c +++ b/Modules/_testcapi/file.c @@ -1,8 +1,43 @@ +// clinic/file.c.h uses internal pycore_modsupport.h API +#define PYTESTCAPI_NEED_INTERNAL_API + #include "parts.h" #include "util.h" +#include "clinic/file.c.h" + +/*[clinic input] +module _testcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ + +/*[clinic input] +_testcapi.py_fopen + + path: object + mode: str + / + +Call Py_fopen(), fread(256) and Py_fclose(). Return read bytes. +[clinic start generated code]*/ +static PyObject * +_testcapi_py_fopen_impl(PyObject *module, PyObject *path, const char *mode) +/*[clinic end generated code: output=5a900af000f759de input=d7e7b8f0fd151953]*/ +{ + FILE *fp = Py_fopen(path, mode); + if (fp == NULL) { + return NULL; + } + + char buffer[256]; + size_t size = fread(buffer, 1, Py_ARRAY_LENGTH(buffer), fp); + Py_fclose(fp); + + return PyBytes_FromStringAndSize(buffer, size); +} static PyMethodDef test_methods[] = { + _TESTCAPI_PY_FOPEN_METHODDEF {NULL}, }; diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 3af5429ef00985..841410c52b3ce2 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -15,7 +15,7 @@ call_pyobject_print(PyObject *self, PyObject * args) return NULL; } - fp = _Py_fopen_obj(filename, "w+"); + fp = Py_fopen(filename, "w+"); if (Py_IsTrue(print_raw)) { flags = Py_PRINT_RAW; @@ -41,7 +41,7 @@ pyobject_print_null(PyObject *self, PyObject *args) return NULL; } - fp = _Py_fopen_obj(filename, "w+"); + fp = Py_fopen(filename, "w+"); if (PyObject_Print(NULL, fp, 0) < 0) { fclose(fp); @@ -72,7 +72,7 @@ pyobject_print_noref_object(PyObject *self, PyObject *args) return NULL; } - fp = _Py_fopen_obj(filename, "w+"); + fp = Py_fopen(filename, "w+"); if (PyObject_Print(test_string, fp, 0) < 0){ fclose(fp); @@ -103,7 +103,7 @@ pyobject_print_os_error(PyObject *self, PyObject *args) } // open file in read mode to induce OSError - fp = _Py_fopen_obj(filename, "r"); + fp = Py_fopen(filename, "r"); if (PyObject_Print(test_string, fp, 0) < 0) { fclose(fp); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f737250ac29d57..cd9014118f2d7f 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1744,7 +1744,7 @@ pymarshal_write_long_to_file(PyObject* self, PyObject *args) &value, &filename, &version)) return NULL; - fp = _Py_fopen_obj(filename, "wb"); + fp = Py_fopen(filename, "wb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1769,7 +1769,7 @@ pymarshal_write_object_to_file(PyObject* self, PyObject *args) &obj, &filename, &version)) return NULL; - fp = _Py_fopen_obj(filename, "wb"); + fp = Py_fopen(filename, "wb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1793,7 +1793,7 @@ pymarshal_read_short_from_file(PyObject* self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pymarshal_read_short_from_file", &filename)) return NULL; - fp = _Py_fopen_obj(filename, "rb"); + fp = Py_fopen(filename, "rb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1818,7 +1818,7 @@ pymarshal_read_long_from_file(PyObject* self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pymarshal_read_long_from_file", &filename)) return NULL; - fp = _Py_fopen_obj(filename, "rb"); + fp = Py_fopen(filename, "rb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1840,7 +1840,7 @@ pymarshal_read_last_object_from_file(PyObject* self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pymarshal_read_last_object_from_file", &filename)) return NULL; - FILE *fp = _Py_fopen_obj(filename, "rb"); + FILE *fp = Py_fopen(filename, "rb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1863,7 +1863,7 @@ pymarshal_read_object_from_file(PyObject* self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pymarshal_read_object_from_file", &filename)) return NULL; - FILE *fp = _Py_fopen_obj(filename, "rb"); + FILE *fp = Py_fopen(filename, "rb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; diff --git a/Modules/main.c b/Modules/main.c index 3bf2241f2837a3..5bb1de2d04d30c 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -370,7 +370,7 @@ pymain_run_file_obj(PyObject *program_name, PyObject *filename, return pymain_exit_err_print(); } - FILE *fp = _Py_fopen_obj(filename, "rb"); + FILE *fp = Py_fopen(filename, "rb"); if (fp == NULL) { // Ignore the OSError PyErr_Clear(); @@ -465,7 +465,7 @@ pymain_run_startup(PyConfig *config, int *exitcode) goto error; } - FILE *fp = _Py_fopen_obj(startup, "r"); + FILE *fp = Py_fopen(startup, "r"); if (fp == NULL) { int save_errno = errno; PyErr_Clear(); diff --git a/Python/errors.c b/Python/errors.c index 2d362c1864ffff..b6ac2f767a283b 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1981,7 +1981,7 @@ _PyErr_ProgramDecodedTextObject(PyObject *filename, int lineno, const char* enco return NULL; } - FILE *fp = _Py_fopen_obj(filename, "r" PY_STDIOTEXTMODE); + FILE *fp = Py_fopen(filename, "r" PY_STDIOTEXTMODE); if (fp == NULL) { PyErr_Clear(); return NULL; diff --git a/Python/fileutils.c b/Python/fileutils.c index 81276651f6df44..6bc3a44c3c1313 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1748,8 +1748,10 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode) } -/* Open a file. Call _wfopen() on Windows, or encode the path to the filesystem - encoding and call fopen() otherwise. +/* Open a file. + + On Windows, if 'path' is a Unicode string, call _wfopen(). Otherwise, encode + the path to the filesystem encoding and call fopen(). Return the new file object on success. Raise an exception and return NULL on error. @@ -1762,32 +1764,32 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode) Release the GIL to call _wfopen() or fopen(). The caller must hold the GIL. */ FILE* -_Py_fopen_obj(PyObject *path, const char *mode) +Py_fopen(PyObject *path, const char *mode) { - FILE *f; - int async_err = 0; -#ifdef MS_WINDOWS - wchar_t wmode[10]; - int usize; - assert(PyGILState_Check()); if (PySys_Audit("open", "Osi", path, mode, 0) < 0) { return NULL; } - if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, - "str file path expected under Windows, got %R", - Py_TYPE(path)); + + FILE *f; + int async_err = 0; + int saved_errno; +#ifdef MS_WINDOWS + PyObject *unicode; + if (!PyUnicode_FSDecoder(path, &unicode)) { return NULL; } - wchar_t *wpath = PyUnicode_AsWideCharString(path, NULL); - if (wpath == NULL) + wchar_t *wpath = PyUnicode_AsWideCharString(unicode, NULL); + Py_DECREF(unicode); + if (wpath == NULL) { return NULL; + } - usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, - wmode, Py_ARRAY_LENGTH(wmode)); + wchar_t wmode[10]; + int usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, + wmode, Py_ARRAY_LENGTH(wmode)); if (usize == 0) { PyErr_SetFromWindowsErr(0); PyMem_Free(wpath); @@ -1796,26 +1798,20 @@ _Py_fopen_obj(PyObject *path, const char *mode) do { Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH f = _wfopen(wpath, wmode); + _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (f == NULL && errno == EINTR && !(async_err = PyErr_CheckSignals())); - int saved_errno = errno; + saved_errno = errno; PyMem_Free(wpath); #else PyObject *bytes; - const char *path_bytes; - - assert(PyGILState_Check()); - - if (!PyUnicode_FSConverter(path, &bytes)) - return NULL; - path_bytes = PyBytes_AS_STRING(bytes); - - if (PySys_Audit("open", "Osi", path, mode, 0) < 0) { - Py_DECREF(bytes); + if (!PyUnicode_FSConverter(path, &bytes)) { return NULL; } + const char *path_bytes = PyBytes_AS_STRING(bytes); do { Py_BEGIN_ALLOW_THREADS @@ -1823,11 +1819,13 @@ _Py_fopen_obj(PyObject *path, const char *mode) Py_END_ALLOW_THREADS } while (f == NULL && errno == EINTR && !(async_err = PyErr_CheckSignals())); - int saved_errno = errno; + saved_errno = errno; Py_DECREF(bytes); #endif - if (async_err) + + if (async_err) { return NULL; + } if (f == NULL) { errno = saved_errno; @@ -1842,6 +1840,27 @@ _Py_fopen_obj(PyObject *path, const char *mode) return f; } + +// Deprecated alias to Py_fopen() kept for backward compatibility +FILE* +_Py_fopen_obj(PyObject *path, const char *mode) +{ + return Py_fopen(path, mode); +} + + +// Call fclose(). +// +// On Windows, files opened by Py_fopen() in the Python DLL must be closed by +// the Python DLL to use the same C runtime version. Otherwise, calling +// fclose() directly can cause undefined behavior. +int +Py_fclose(FILE *file) +{ + return fclose(file); +} + + /* Read count bytes from fd into buf. On success, return the number of read bytes, it can be lower than count. diff --git a/Python/import.c b/Python/import.c index a9282dde633959..b3648e24d0e064 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4688,7 +4688,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) * code relies on fp still being open. */ FILE *fp; if (file != NULL) { - fp = _Py_fopen_obj(info.filename, "r"); + fp = Py_fopen(info.filename, "r"); if (fp == NULL) { goto finally; } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 31e065ff00d59a..0da26ad3f9b4bd 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -467,7 +467,7 @@ _PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, fclose(fp); } - pyc_fp = _Py_fopen_obj(filename, "rb"); + pyc_fp = Py_fopen(filename, "rb"); if (pyc_fp == NULL) { fprintf(stderr, "python: Can't reopen .pyc file\n"); goto done; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d6719f9bb0af91..887591a681b25c 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2356,7 +2356,7 @@ static PyObject * sys__dump_tracelets_impl(PyObject *module, PyObject *outpath) /*[clinic end generated code: output=a7fe265e2bc3b674 input=5bff6880cd28ffd1]*/ { - FILE *out = _Py_fopen_obj(outpath, "wb"); + FILE *out = Py_fopen(outpath, "wb"); if (out == NULL) { return NULL; } From b9c693dcca01537eee1ef716ffebc632be37594b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 6 Jan 2025 14:16:22 +0000 Subject: [PATCH 385/427] GH-128073: Include `EXIT_IF` when checking for escaping calls (GH-128537) --- Lib/test/test_generated_cases.py | 25 +++++++++++++++++++++++++ Tools/cases_generator/analyzer.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 9c65e81dfe4be1..75cbd8dd94e9cb 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1713,6 +1713,31 @@ def test_pop_dead_inputs_with_output(self): """ self.run_cases_test(input, output) + def test_no_escaping_calls_in_branching_macros(self): + + input = """ + inst(OP, ( -- )) { + DEOPT_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + input = """ + inst(OP, ( -- )) { + EXIT_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + input = """ + inst(OP, ( -- )) { + ERROR_IF(escaping_call(), error); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index eca851e6de87ae..73c871759afbf5 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -668,7 +668,7 @@ def check_escaping_calls(instr: parser.InstDef, escapes: dict[lexer.Token, tuple if tkn.kind == "IF": next(tkn_iter) in_if = 1 - if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF"): + if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF", "EXIT_IF"): next(tkn_iter) in_if = 1 elif tkn.kind == "LPAREN" and in_if: From f826beca0cedb8e4b92896544c75fd0d9dcb0446 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 6 Jan 2025 17:54:47 +0000 Subject: [PATCH 386/427] GH-128375: Better instrument for `FOR_ITER` (GH-128445) --- Include/internal/pycore_instruments.h | 4 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 39 +- Include/internal/pycore_uop_ids.h | 1 + Include/internal/pycore_uop_metadata.h | 4 + Include/opcode_ids.h | 182 ++++---- Lib/_opcode_metadata.py | 182 ++++---- Lib/test/test_code.py | 40 ++ Lib/test/test_compiler_codegen.py | 3 +- Lib/test/test_dis.py | 392 +++++++++--------- Lib/test/test_monitoring.py | 92 +++- Modules/_testcapimodule.c | 21 + Programs/test_frozenmain.h | 66 +-- Python/bytecodes.c | 57 +-- Python/ceval_macros.h | 2 +- Python/codegen.c | 6 +- Python/executor_cases.c.h | 14 + Python/generated_cases.c.h | 71 ++-- Python/instrumentation.c | 151 ++++--- Python/opcode_targets.h | 4 +- Python/optimizer.c | 12 +- Python/optimizer_cases.c.h | 6 + Tools/cases_generator/analyzer.py | 4 + Tools/cases_generator/generators_common.py | 2 + Tools/cases_generator/lexer.py | 1 + .../opcode_metadata_generator.py | 5 +- Tools/cases_generator/tier1_generator.py | 7 +- 27 files changed, 827 insertions(+), 544 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 4e5b374968ea98..92d8f056f402fc 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -48,8 +48,8 @@ _Py_call_instrumentation_instruction( _Py_CODEUNIT * _Py_call_instrumentation_jump( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target); + _Py_CODEUNIT *instr, PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest); extern int _Py_call_instrumentation_arg(PyThreadState *tstate, int event, diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index ec3685d2034560..f9f71d7453331e 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -264,6 +264,7 @@ Known values: Python 3.14a2 3609 (Add LOAD_SMALL_INT and LOAD_CONST_IMMORTAL instructions, remove RETURN_CONST) Python 3.14a4 3610 (Add VALUE_WITH_FAKE_GLOBALS format to annotationlib) Python 3.14a4 3611 (Add NOT_TAKEN instruction) + Python 3.14a4 3612 (Add POP_ITER and INSTRUMENTED_POP_ITER) Python 3.15 will start with 3650 @@ -276,7 +277,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3611 +#define PYC_MAGIC_NUMBER 3612 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 5fb236836dccd9..90d5e277d8d6ce 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -245,6 +245,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case INSTRUMENTED_NOT_TAKEN: return 0; + case INSTRUMENTED_POP_ITER: + return 1; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -375,6 +377,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case POP_EXCEPT: return 1; + case POP_ITER: + return 1; case POP_JUMP_IF_FALSE: return 1; case POP_JUMP_IF_NONE: @@ -708,6 +712,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case INSTRUMENTED_NOT_TAKEN: return 0; + case INSTRUMENTED_POP_ITER: + return 0; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -838,6 +844,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case POP_EXCEPT: return 0; + case POP_ITER: + return 0; case POP_JUMP_IF_FALSE: return 0; case POP_JUMP_IF_NONE: @@ -1399,6 +1407,10 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { *effect = 0; return 0; } + case INSTRUMENTED_POP_ITER: { + *effect = -1; + return 0; + } case INSTRUMENTED_POP_JUMP_IF_FALSE: { *effect = 0; return 0; @@ -1659,6 +1671,10 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { *effect = -1; return 0; } + case POP_ITER: { + *effect = -1; + return 0; + } case POP_JUMP_IF_FALSE: { *effect = -1; return 0; @@ -1921,6 +1937,7 @@ enum InstructionFormat { #define HAS_PASSTHROUGH_FLAG (4096) #define HAS_OPARG_AND_1_FLAG (8192) #define HAS_ERROR_NO_POP_FLAG (16384) +#define HAS_NO_SAVE_IP_FLAG (32768) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1936,6 +1953,7 @@ enum InstructionFormat { #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) +#define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -1948,8 +1966,8 @@ enum InstructionFormat { struct opcode_metadata { uint8_t valid_entry; - int8_t instr_format; - int16_t flags; + uint8_t instr_format; + uint16_t flags; }; extern const struct opcode_metadata _PyOpcode_opcode_metadata[266]; @@ -2028,7 +2046,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [END_FOR] = { true, INSTR_FMT_IX, HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -2051,15 +2069,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IXC, 0 }, [INSTRUMENTED_NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_POP_ITER] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, @@ -2119,6 +2138,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [NOT_TAKEN] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [POP_ITER] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -2261,7 +2281,7 @@ _PyOpcode_macro_expansion[256] = { [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, 0, 0 } } }, [DICT_MERGE] = { .nuops = 1, .uops = { { _DICT_MERGE, 0, 0 } } }, [DICT_UPDATE] = { .nuops = 1, .uops = { { _DICT_UPDATE, 0, 0 } } }, - [END_FOR] = { .nuops = 1, .uops = { { _POP_TOP, 0, 0 } } }, + [END_FOR] = { .nuops = 1, .uops = { { _END_FOR, 0, 0 } } }, [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, 0, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, 0, 0 } } }, [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, 0, 0 } } }, @@ -2324,6 +2344,7 @@ _PyOpcode_macro_expansion[256] = { [NOP] = { .nuops = 1, .uops = { { _NOP, 0, 0 } } }, [NOT_TAKEN] = { .nuops = 1, .uops = { { _NOP, 0, 0 } } }, [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, 0, 0 } } }, + [POP_ITER] = { .nuops = 1, .uops = { { _POP_TOP, 0, 0 } } }, [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 9, 1 } } }, [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 9, 1 } } }, [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_FALSE, 9, 1 } } }, @@ -2482,6 +2503,7 @@ const char *_PyOpcode_OpName[266] = { [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", [INSTRUMENTED_NOT_TAKEN] = "INSTRUMENTED_NOT_TAKEN", + [INSTRUMENTED_POP_ITER] = "INSTRUMENTED_POP_ITER", [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", @@ -2547,6 +2569,7 @@ const char *_PyOpcode_OpName[266] = { [NOT_TAKEN] = "NOT_TAKEN", [POP_BLOCK] = "POP_BLOCK", [POP_EXCEPT] = "POP_EXCEPT", + [POP_ITER] = "POP_ITER", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", @@ -2740,6 +2763,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, [INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_ITER] = INSTRUMENTED_POP_ITER, [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, @@ -2799,6 +2823,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [NOP] = NOP, [NOT_TAKEN] = NOT_TAKEN, [POP_EXCEPT] = POP_EXCEPT, + [POP_ITER] = POP_ITER, [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, @@ -2856,7 +2881,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 117: \ case 118: \ case 119: \ case 120: \ @@ -2895,7 +2919,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 232: \ case 233: \ case 234: \ - case 235: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 92515b4230ccb4..21690a28839565 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -101,6 +101,7 @@ extern "C" { #define _DO_CALL_FUNCTION_EX 358 #define _DO_CALL_KW 359 #define _DYNAMIC_EXIT 360 +#define _END_FOR END_FOR #define _END_SEND END_SEND #define _ERROR_POP_N 361 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 73fc29eb78a7a4..83e578cdd76fbd 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -55,6 +55,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_POP_TOP] = HAS_PURE_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, + [_END_FOR] = HAS_NO_SAVE_IP_FLAG, [_END_SEND] = HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, @@ -391,6 +392,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", + [_END_FOR] = "_END_FOR", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", @@ -655,6 +657,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _PUSH_NULL: return 0; + case _END_FOR: + return 1; case _END_SEND: return 2; case _UNARY_NEGATIVE: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 3cd189b93dd9d6..09e261fadd5544 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -40,93 +40,94 @@ extern "C" { #define NOP 27 #define NOT_TAKEN 28 #define POP_EXCEPT 29 -#define POP_TOP 30 -#define PUSH_EXC_INFO 31 -#define PUSH_NULL 32 -#define RETURN_GENERATOR 33 -#define RETURN_VALUE 34 -#define SETUP_ANNOTATIONS 35 -#define STORE_SLICE 36 -#define STORE_SUBSCR 37 -#define TO_BOOL 38 -#define UNARY_INVERT 39 -#define UNARY_NEGATIVE 40 -#define UNARY_NOT 41 -#define WITH_EXCEPT_START 42 -#define BINARY_OP 43 -#define BUILD_LIST 44 -#define BUILD_MAP 45 -#define BUILD_SET 46 -#define BUILD_SLICE 47 -#define BUILD_STRING 48 -#define BUILD_TUPLE 49 -#define CALL 50 -#define CALL_FUNCTION_EX 51 -#define CALL_INTRINSIC_1 52 -#define CALL_INTRINSIC_2 53 -#define CALL_KW 54 -#define COMPARE_OP 55 -#define CONTAINS_OP 56 -#define CONVERT_VALUE 57 -#define COPY 58 -#define COPY_FREE_VARS 59 -#define DELETE_ATTR 60 -#define DELETE_DEREF 61 -#define DELETE_FAST 62 -#define DELETE_GLOBAL 63 -#define DELETE_NAME 64 -#define DICT_MERGE 65 -#define DICT_UPDATE 66 -#define EXTENDED_ARG 67 -#define FOR_ITER 68 -#define GET_AWAITABLE 69 -#define IMPORT_FROM 70 -#define IMPORT_NAME 71 -#define IS_OP 72 -#define JUMP_BACKWARD 73 -#define JUMP_BACKWARD_NO_INTERRUPT 74 -#define JUMP_FORWARD 75 -#define LIST_APPEND 76 -#define LIST_EXTEND 77 -#define LOAD_ATTR 78 -#define LOAD_COMMON_CONSTANT 79 -#define LOAD_CONST 80 -#define LOAD_DEREF 81 -#define LOAD_FAST 82 -#define LOAD_FAST_AND_CLEAR 83 -#define LOAD_FAST_CHECK 84 -#define LOAD_FAST_LOAD_FAST 85 -#define LOAD_FROM_DICT_OR_DEREF 86 -#define LOAD_FROM_DICT_OR_GLOBALS 87 -#define LOAD_GLOBAL 88 -#define LOAD_NAME 89 -#define LOAD_SMALL_INT 90 -#define LOAD_SPECIAL 91 -#define LOAD_SUPER_ATTR 92 -#define MAKE_CELL 93 -#define MAP_ADD 94 -#define MATCH_CLASS 95 -#define POP_JUMP_IF_FALSE 96 -#define POP_JUMP_IF_NONE 97 -#define POP_JUMP_IF_NOT_NONE 98 -#define POP_JUMP_IF_TRUE 99 -#define RAISE_VARARGS 100 -#define RERAISE 101 -#define SEND 102 -#define SET_ADD 103 -#define SET_FUNCTION_ATTRIBUTE 104 -#define SET_UPDATE 105 -#define STORE_ATTR 106 -#define STORE_DEREF 107 -#define STORE_FAST 108 -#define STORE_FAST_LOAD_FAST 109 -#define STORE_FAST_STORE_FAST 110 -#define STORE_GLOBAL 111 -#define STORE_NAME 112 -#define SWAP 113 -#define UNPACK_EX 114 -#define UNPACK_SEQUENCE 115 -#define YIELD_VALUE 116 +#define POP_ITER 30 +#define POP_TOP 31 +#define PUSH_EXC_INFO 32 +#define PUSH_NULL 33 +#define RETURN_GENERATOR 34 +#define RETURN_VALUE 35 +#define SETUP_ANNOTATIONS 36 +#define STORE_SLICE 37 +#define STORE_SUBSCR 38 +#define TO_BOOL 39 +#define UNARY_INVERT 40 +#define UNARY_NEGATIVE 41 +#define UNARY_NOT 42 +#define WITH_EXCEPT_START 43 +#define BINARY_OP 44 +#define BUILD_LIST 45 +#define BUILD_MAP 46 +#define BUILD_SET 47 +#define BUILD_SLICE 48 +#define BUILD_STRING 49 +#define BUILD_TUPLE 50 +#define CALL 51 +#define CALL_FUNCTION_EX 52 +#define CALL_INTRINSIC_1 53 +#define CALL_INTRINSIC_2 54 +#define CALL_KW 55 +#define COMPARE_OP 56 +#define CONTAINS_OP 57 +#define CONVERT_VALUE 58 +#define COPY 59 +#define COPY_FREE_VARS 60 +#define DELETE_ATTR 61 +#define DELETE_DEREF 62 +#define DELETE_FAST 63 +#define DELETE_GLOBAL 64 +#define DELETE_NAME 65 +#define DICT_MERGE 66 +#define DICT_UPDATE 67 +#define EXTENDED_ARG 68 +#define FOR_ITER 69 +#define GET_AWAITABLE 70 +#define IMPORT_FROM 71 +#define IMPORT_NAME 72 +#define IS_OP 73 +#define JUMP_BACKWARD 74 +#define JUMP_BACKWARD_NO_INTERRUPT 75 +#define JUMP_FORWARD 76 +#define LIST_APPEND 77 +#define LIST_EXTEND 78 +#define LOAD_ATTR 79 +#define LOAD_COMMON_CONSTANT 80 +#define LOAD_CONST 81 +#define LOAD_DEREF 82 +#define LOAD_FAST 83 +#define LOAD_FAST_AND_CLEAR 84 +#define LOAD_FAST_CHECK 85 +#define LOAD_FAST_LOAD_FAST 86 +#define LOAD_FROM_DICT_OR_DEREF 87 +#define LOAD_FROM_DICT_OR_GLOBALS 88 +#define LOAD_GLOBAL 89 +#define LOAD_NAME 90 +#define LOAD_SMALL_INT 91 +#define LOAD_SPECIAL 92 +#define LOAD_SUPER_ATTR 93 +#define MAKE_CELL 94 +#define MAP_ADD 95 +#define MATCH_CLASS 96 +#define POP_JUMP_IF_FALSE 97 +#define POP_JUMP_IF_NONE 98 +#define POP_JUMP_IF_NOT_NONE 99 +#define POP_JUMP_IF_TRUE 100 +#define RAISE_VARARGS 101 +#define RERAISE 102 +#define SEND 103 +#define SET_ADD 104 +#define SET_FUNCTION_ATTRIBUTE 105 +#define SET_UPDATE 106 +#define STORE_ATTR 107 +#define STORE_DEREF 108 +#define STORE_FAST 109 +#define STORE_FAST_LOAD_FAST 110 +#define STORE_FAST_STORE_FAST 111 +#define STORE_GLOBAL 112 +#define STORE_NAME 113 +#define SWAP 114 +#define UNPACK_EX 115 +#define UNPACK_SEQUENCE 116 +#define YIELD_VALUE 117 #define RESUME 149 #define BINARY_OP_ADD_FLOAT 150 #define BINARY_OP_ADD_INT 151 @@ -206,7 +207,8 @@ extern "C" { #define UNPACK_SEQUENCE_LIST 225 #define UNPACK_SEQUENCE_TUPLE 226 #define UNPACK_SEQUENCE_TWO_TUPLE 227 -#define INSTRUMENTED_END_FOR 236 +#define INSTRUMENTED_END_FOR 235 +#define INSTRUMENTED_POP_ITER 236 #define INSTRUMENTED_END_SEND 237 #define INSTRUMENTED_LOAD_SUPER_ATTR 238 #define INSTRUMENTED_FOR_ITER 239 @@ -237,9 +239,9 @@ extern "C" { #define SETUP_WITH 264 #define STORE_FAST_MAYBE_NULL 265 -#define HAVE_ARGUMENT 42 +#define HAVE_ARGUMENT 43 #define MIN_SPECIALIZED_OPCODE 150 -#define MIN_INSTRUMENTED_OPCODE 236 +#define MIN_INSTRUMENTED_OPCODE 235 #ifdef __cplusplus } diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index dada2cb5fa033f..64ee56fd10556f 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -233,94 +233,96 @@ 'NOP': 27, 'NOT_TAKEN': 28, 'POP_EXCEPT': 29, - 'POP_TOP': 30, - 'PUSH_EXC_INFO': 31, - 'PUSH_NULL': 32, - 'RETURN_GENERATOR': 33, - 'RETURN_VALUE': 34, - 'SETUP_ANNOTATIONS': 35, - 'STORE_SLICE': 36, - 'STORE_SUBSCR': 37, - 'TO_BOOL': 38, - 'UNARY_INVERT': 39, - 'UNARY_NEGATIVE': 40, - 'UNARY_NOT': 41, - 'WITH_EXCEPT_START': 42, - 'BINARY_OP': 43, - 'BUILD_LIST': 44, - 'BUILD_MAP': 45, - 'BUILD_SET': 46, - 'BUILD_SLICE': 47, - 'BUILD_STRING': 48, - 'BUILD_TUPLE': 49, - 'CALL': 50, - 'CALL_FUNCTION_EX': 51, - 'CALL_INTRINSIC_1': 52, - 'CALL_INTRINSIC_2': 53, - 'CALL_KW': 54, - 'COMPARE_OP': 55, - 'CONTAINS_OP': 56, - 'CONVERT_VALUE': 57, - 'COPY': 58, - 'COPY_FREE_VARS': 59, - 'DELETE_ATTR': 60, - 'DELETE_DEREF': 61, - 'DELETE_FAST': 62, - 'DELETE_GLOBAL': 63, - 'DELETE_NAME': 64, - 'DICT_MERGE': 65, - 'DICT_UPDATE': 66, - 'EXTENDED_ARG': 67, - 'FOR_ITER': 68, - 'GET_AWAITABLE': 69, - 'IMPORT_FROM': 70, - 'IMPORT_NAME': 71, - 'IS_OP': 72, - 'JUMP_BACKWARD': 73, - 'JUMP_BACKWARD_NO_INTERRUPT': 74, - 'JUMP_FORWARD': 75, - 'LIST_APPEND': 76, - 'LIST_EXTEND': 77, - 'LOAD_ATTR': 78, - 'LOAD_COMMON_CONSTANT': 79, - 'LOAD_CONST': 80, - 'LOAD_DEREF': 81, - 'LOAD_FAST': 82, - 'LOAD_FAST_AND_CLEAR': 83, - 'LOAD_FAST_CHECK': 84, - 'LOAD_FAST_LOAD_FAST': 85, - 'LOAD_FROM_DICT_OR_DEREF': 86, - 'LOAD_FROM_DICT_OR_GLOBALS': 87, - 'LOAD_GLOBAL': 88, - 'LOAD_NAME': 89, - 'LOAD_SMALL_INT': 90, - 'LOAD_SPECIAL': 91, - 'LOAD_SUPER_ATTR': 92, - 'MAKE_CELL': 93, - 'MAP_ADD': 94, - 'MATCH_CLASS': 95, - 'POP_JUMP_IF_FALSE': 96, - 'POP_JUMP_IF_NONE': 97, - 'POP_JUMP_IF_NOT_NONE': 98, - 'POP_JUMP_IF_TRUE': 99, - 'RAISE_VARARGS': 100, - 'RERAISE': 101, - 'SEND': 102, - 'SET_ADD': 103, - 'SET_FUNCTION_ATTRIBUTE': 104, - 'SET_UPDATE': 105, - 'STORE_ATTR': 106, - 'STORE_DEREF': 107, - 'STORE_FAST': 108, - 'STORE_FAST_LOAD_FAST': 109, - 'STORE_FAST_STORE_FAST': 110, - 'STORE_GLOBAL': 111, - 'STORE_NAME': 112, - 'SWAP': 113, - 'UNPACK_EX': 114, - 'UNPACK_SEQUENCE': 115, - 'YIELD_VALUE': 116, - 'INSTRUMENTED_END_FOR': 236, + 'POP_ITER': 30, + 'POP_TOP': 31, + 'PUSH_EXC_INFO': 32, + 'PUSH_NULL': 33, + 'RETURN_GENERATOR': 34, + 'RETURN_VALUE': 35, + 'SETUP_ANNOTATIONS': 36, + 'STORE_SLICE': 37, + 'STORE_SUBSCR': 38, + 'TO_BOOL': 39, + 'UNARY_INVERT': 40, + 'UNARY_NEGATIVE': 41, + 'UNARY_NOT': 42, + 'WITH_EXCEPT_START': 43, + 'BINARY_OP': 44, + 'BUILD_LIST': 45, + 'BUILD_MAP': 46, + 'BUILD_SET': 47, + 'BUILD_SLICE': 48, + 'BUILD_STRING': 49, + 'BUILD_TUPLE': 50, + 'CALL': 51, + 'CALL_FUNCTION_EX': 52, + 'CALL_INTRINSIC_1': 53, + 'CALL_INTRINSIC_2': 54, + 'CALL_KW': 55, + 'COMPARE_OP': 56, + 'CONTAINS_OP': 57, + 'CONVERT_VALUE': 58, + 'COPY': 59, + 'COPY_FREE_VARS': 60, + 'DELETE_ATTR': 61, + 'DELETE_DEREF': 62, + 'DELETE_FAST': 63, + 'DELETE_GLOBAL': 64, + 'DELETE_NAME': 65, + 'DICT_MERGE': 66, + 'DICT_UPDATE': 67, + 'EXTENDED_ARG': 68, + 'FOR_ITER': 69, + 'GET_AWAITABLE': 70, + 'IMPORT_FROM': 71, + 'IMPORT_NAME': 72, + 'IS_OP': 73, + 'JUMP_BACKWARD': 74, + 'JUMP_BACKWARD_NO_INTERRUPT': 75, + 'JUMP_FORWARD': 76, + 'LIST_APPEND': 77, + 'LIST_EXTEND': 78, + 'LOAD_ATTR': 79, + 'LOAD_COMMON_CONSTANT': 80, + 'LOAD_CONST': 81, + 'LOAD_DEREF': 82, + 'LOAD_FAST': 83, + 'LOAD_FAST_AND_CLEAR': 84, + 'LOAD_FAST_CHECK': 85, + 'LOAD_FAST_LOAD_FAST': 86, + 'LOAD_FROM_DICT_OR_DEREF': 87, + 'LOAD_FROM_DICT_OR_GLOBALS': 88, + 'LOAD_GLOBAL': 89, + 'LOAD_NAME': 90, + 'LOAD_SMALL_INT': 91, + 'LOAD_SPECIAL': 92, + 'LOAD_SUPER_ATTR': 93, + 'MAKE_CELL': 94, + 'MAP_ADD': 95, + 'MATCH_CLASS': 96, + 'POP_JUMP_IF_FALSE': 97, + 'POP_JUMP_IF_NONE': 98, + 'POP_JUMP_IF_NOT_NONE': 99, + 'POP_JUMP_IF_TRUE': 100, + 'RAISE_VARARGS': 101, + 'RERAISE': 102, + 'SEND': 103, + 'SET_ADD': 104, + 'SET_FUNCTION_ATTRIBUTE': 105, + 'SET_UPDATE': 106, + 'STORE_ATTR': 107, + 'STORE_DEREF': 108, + 'STORE_FAST': 109, + 'STORE_FAST_LOAD_FAST': 110, + 'STORE_FAST_STORE_FAST': 111, + 'STORE_GLOBAL': 112, + 'STORE_NAME': 113, + 'SWAP': 114, + 'UNPACK_EX': 115, + 'UNPACK_SEQUENCE': 116, + 'YIELD_VALUE': 117, + 'INSTRUMENTED_END_FOR': 235, + 'INSTRUMENTED_POP_ITER': 236, 'INSTRUMENTED_END_SEND': 237, 'INSTRUMENTED_LOAD_SUPER_ATTR': 238, 'INSTRUMENTED_FOR_ITER': 239, @@ -350,5 +352,5 @@ 'STORE_FAST_MAYBE_NULL': 265, } -HAVE_ARGUMENT = 42 -MIN_INSTRUMENTED_OPCODE = 236 +HAVE_ARGUMENT = 43 +MIN_INSTRUMENTED_OPCODE = 235 diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 2a1b26e8a1ffd1..7ffa4eb8639add 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -215,6 +215,8 @@ from test.support import threading_helper, import_helper from test.support.bytecode_helper import instructions_with_positions from opcode import opmap, opname +from _testcapi import code_offset_to_line + COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -896,6 +898,44 @@ async def async_func(): rc, out, err = assert_python_ok('-OO', '-c', code) + def test_co_branches(self): + + def get_line_branches(func): + code = func.__code__ + base = code.co_firstlineno + return [ + ( + code_offset_to_line(code, src) - base, + code_offset_to_line(code, left) - base, + code_offset_to_line(code, right) - base + ) for (src, left, right) in + code.co_branches() + ] + + def simple(x): + if x: + A + else: + B + + self.assertEqual( + get_line_branches(simple), + [(1,2,4)]) + + def with_extended_args(x): + if x: + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + else: + B + + self.assertEqual( + get_line_branches(with_extended_args), + [(1,2,8)]) + if check_impl_detail(cpython=True) and ctypes is not None: py = ctypes.pythonapi freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index f8c4fc14c91ebe..5655e1b9cf196a 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -50,7 +50,6 @@ def test_for_loop(self): ('GET_ITER', None, 1), loop_lbl := self.Label(), ('FOR_ITER', exit_lbl := self.Label(), 1), - ('NOT_TAKEN', None, 1), ('NOP', None, 1, 1), ('STORE_NAME', 1, 1), ('LOAD_NAME', 2, 2), @@ -61,7 +60,7 @@ def test_for_loop(self): ('JUMP', loop_lbl), exit_lbl, ('END_FOR', None), - ('POP_TOP', None), + ('POP_ITER', None), ('LOAD_CONST', 0), ('RETURN_VALUE', None), ] diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 955a3e4cb9e4f7..e733a673d003e7 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -175,14 +175,13 @@ def bug708901(): %3d CALL 2 GET_ITER - L1: FOR_ITER 4 (to L2) - NOT_TAKEN + L1: FOR_ITER 3 (to L2) STORE_FAST 0 (res) -%3d JUMP_BACKWARD 6 (to L1) +%3d JUMP_BACKWARD 5 (to L1) %3d L2: END_FOR - POP_TOP + POP_ITER LOAD_CONST 0 (None) RETURN_VALUE """ % (bug708901.__code__.co_firstlineno, @@ -844,8 +843,7 @@ def foo(x): L1: RESUME 0 LOAD_FAST 0 (.0) GET_ITER - L2: FOR_ITER 11 (to L3) - NOT_TAKEN + L2: FOR_ITER 10 (to L3) STORE_FAST 1 (z) LOAD_DEREF 2 (x) LOAD_FAST 1 (z) @@ -853,9 +851,9 @@ def foo(x): YIELD_VALUE 0 RESUME 5 POP_TOP - JUMP_BACKWARD 13 (to L2) + JUMP_BACKWARD 12 (to L2) L3: END_FOR - POP_TOP + POP_ITER LOAD_CONST 0 (None) RETURN_VALUE @@ -899,18 +897,17 @@ def loop_test(): LOAD_SMALL_INT 3 BINARY_OP 5 (*) GET_ITER - L1: FOR_ITER_LIST 15 (to L2) - NOT_TAKEN + L1: FOR_ITER_LIST 14 (to L2) STORE_FAST 0 (i) %3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST 0 (i) CALL_PY_GENERAL 1 POP_TOP - JUMP_BACKWARD 17 (to L1) + JUMP_BACKWARD 16 (to L1) %3d L2: END_FOR - POP_TOP + POP_ITER LOAD_CONST_IMMORTAL 1 (None) RETURN_VALUE """ % (loop_test.__code__.co_firstlineno, @@ -1706,214 +1703,213 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=93, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=93, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_TUPLE', opcode=49, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=50, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=108, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=81, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=81, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=1, argval=1, argrepr='', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_LIST', opcode=44, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_MAP', opcode=45, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=2, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=109, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=82, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_LIST', opcode=45, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_MAP', opcode=46, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=59, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=93, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=93, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='COPY_FREE_VARS', opcode=60, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_TUPLE', opcode=49, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=50, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=108, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=81, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=81, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=81, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=81, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=109, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=82, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=59, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='COPY_FREE_VARS', opcode=60, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=81, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=81, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=81, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=81, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=85, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=36, start_offset=36, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=82, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=86, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=36, start_offset=36, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='GET_ITER', opcode=16, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='FOR_ITER', opcode=68, arg=34, argval=96, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=108, arg=0, argval='i', argrepr='i', offset=30, start_offset=30, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=32, start_offset=32, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=52, start_offset=52, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=54, start_offset=54, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=4, argval=4, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=55, arg=18, argval='<', argrepr='bool(<)', offset=58, start_offset=58, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=3, argval=72, argrepr='to L2', offset=62, start_offset=62, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=66, start_offset=66, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD', opcode=73, arg=24, argval=24, argrepr='to L1', offset=68, start_offset=68, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=72, start_offset=72, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=6, argval=6, argrepr='', offset=74, start_offset=74, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=55, arg=148, argval='>', argrepr='bool(>)', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=3, argval=90, argrepr='to L3', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD', opcode=73, arg=33, argval=24, argrepr='to L1', offset=86, start_offset=86, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=True, line_number=None, label=3, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=92, start_offset=92, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_FORWARD', opcode=75, arg=13, argval=122, argrepr='to L5', offset=94, start_offset=94, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=96, start_offset=96, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=98, start_offset=98, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=100, start_offset=100, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=110, start_offset=110, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=120, start_offset=120, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=84, arg=0, argval='i', argrepr='i', offset=122, start_offset=122, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=38, arg=None, argval=None, argrepr='', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=37, argval=210, argrepr='to L8', offset=132, start_offset=132, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=136, start_offset=136, starts_line=False, line_number=11, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=138, start_offset=138, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=150, start_offset=150, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=158, start_offset=158, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=1, argval=1, argrepr='', offset=162, start_offset=162, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=43, arg=23, argval=23, argrepr='-=', offset=164, start_offset=164, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=108, arg=0, argval='i', argrepr='i', offset=168, start_offset=168, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=170, start_offset=170, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=6, argval=6, argrepr='', offset=172, start_offset=172, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=55, arg=148, argval='>', argrepr='bool(>)', offset=174, start_offset=174, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=3, argval=188, argrepr='to L6', offset=178, start_offset=178, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=182, start_offset=182, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD', opcode=73, arg=33, argval=122, argrepr='to L5', offset=184, start_offset=184, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=188, start_offset=188, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=4, argval=4, argrepr='', offset=190, start_offset=190, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=55, arg=18, argval='<', argrepr='bool(<)', offset=192, start_offset=192, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=3, argval=206, argrepr='to L7', offset=196, start_offset=196, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=200, start_offset=200, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD', opcode=73, arg=42, argval=122, argrepr='to L5', offset=202, start_offset=202, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=206, start_offset=206, starts_line=True, line_number=None, label=7, positions=None, cache_info=None), - Instruction(opname='JUMP_FORWARD', opcode=75, arg=11, argval=232, argrepr='to L9', offset=208, start_offset=208, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=210, start_offset=210, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=222, start_offset=222, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=232, start_offset=232, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=1, argval=1, argrepr='', offset=234, start_offset=234, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=90, arg=0, argval=0, argrepr='', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=43, arg=11, argval=11, argrepr='/', offset=238, start_offset=238, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=242, start_offset=242, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=244, start_offset=244, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=58, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=91, arg=1, argval=1, argrepr='__exit__', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=113, arg=2, argval=2, argrepr='', offset=250, start_offset=250, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=113, arg=3, argval=3, argrepr='', offset=252, start_offset=252, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=91, arg=0, argval=0, argrepr='__enter__', offset=254, start_offset=254, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=0, argval=0, argrepr='', offset=256, start_offset=256, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=108, arg=1, argval='dodgy', argrepr='dodgy', offset=264, start_offset=264, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=266, start_offset=266, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=80, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=276, start_offset=276, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=278, start_offset=278, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=286, start_offset=286, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=288, start_offset=288, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=290, start_offset=290, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=292, start_offset=292, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=3, argval=3, argrepr='', offset=294, start_offset=294, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=302, start_offset=302, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=304, start_offset=304, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=80, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=314, start_offset=314, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=316, start_offset=316, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=324, start_offset=324, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=None, argrepr='None', offset=326, start_offset=326, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=31, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='WITH_EXCEPT_START', opcode=42, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=38, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=2, argval=350, argrepr='to L11', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=2, argval=2, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=352, start_offset=352, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=354, start_offset=354, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=74, arg=29, argval=304, argrepr='to L10', offset=360, start_offset=360, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=362, start_offset=362, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=31, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=370, start_offset=370, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=15, argval=416, argrepr='to L12', offset=382, start_offset=382, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=388, start_offset=388, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=390, start_offset=390, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=80, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=400, start_offset=400, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=402, start_offset=402, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=410, start_offset=410, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=74, arg=56, argval=304, argrepr='to L10', offset=414, start_offset=414, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=0, argval=0, argrepr='', offset=416, start_offset=416, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=418, start_offset=418, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=31, arg=None, argval=None, argrepr='', offset=424, start_offset=424, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=426, start_offset=426, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=80, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=436, start_offset=436, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=438, start_offset=438, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=30, arg=None, argval=None, argrepr='', offset=446, start_offset=446, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=0, argval=0, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=450, start_offset=450, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=452, start_offset=452, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=454, start_offset=454, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='FOR_ITER', opcode=69, arg=33, argval=94, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=4, argval=4, argrepr='', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=3, argval=70, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=64, start_offset=64, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=23, argval=24, argrepr='to L1', offset=66, start_offset=66, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=70, start_offset=70, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=6, argval=6, argrepr='', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=148, argval='>', argrepr='bool(>)', offset=74, start_offset=74, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=3, argval=88, argrepr='to L3', offset=78, start_offset=78, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=82, start_offset=82, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=32, argval=24, argrepr='to L1', offset=84, start_offset=84, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=None, label=3, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=76, arg=13, argval=120, argrepr='to L5', offset=92, start_offset=92, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), + Instruction(opname='POP_ITER', opcode=30, arg=None, argval=None, argrepr='', offset=96, start_offset=96, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=98, start_offset=98, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=108, start_offset=108, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=118, start_offset=118, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=85, arg=0, argval='i', argrepr='i', offset=120, start_offset=120, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=122, start_offset=122, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=37, argval=208, argrepr='to L8', offset=130, start_offset=130, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=134, start_offset=134, starts_line=False, line_number=11, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=136, start_offset=136, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=146, start_offset=146, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=156, start_offset=156, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=160, start_offset=160, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=44, arg=23, argval=23, argrepr='-=', offset=162, start_offset=162, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=168, start_offset=168, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=6, argval=6, argrepr='', offset=170, start_offset=170, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=148, argval='>', argrepr='bool(>)', offset=172, start_offset=172, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=3, argval=186, argrepr='to L6', offset=176, start_offset=176, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=33, argval=120, argrepr='to L5', offset=182, start_offset=182, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=186, start_offset=186, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=4, argval=4, argrepr='', offset=188, start_offset=188, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=18, argval='<', argrepr='bool(<)', offset=190, start_offset=190, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=3, argval=204, argrepr='to L7', offset=194, start_offset=194, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=198, start_offset=198, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=42, argval=120, argrepr='to L5', offset=200, start_offset=200, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=204, start_offset=204, starts_line=True, line_number=None, label=7, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=76, arg=11, argval=230, argrepr='to L9', offset=206, start_offset=206, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=208, start_offset=208, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=232, start_offset=232, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=0, argval=0, argrepr='', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=44, arg=11, argval=11, argrepr='/', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=242, start_offset=242, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=1, argval=1, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=92, arg=1, argval=1, argrepr='__exit__', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=114, arg=2, argval=2, argrepr='', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=114, arg=3, argval=3, argrepr='', offset=250, start_offset=250, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=92, arg=0, argval=0, argrepr='__enter__', offset=252, start_offset=252, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=0, argval=0, argrepr='', offset=254, start_offset=254, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=1, argval='dodgy', argrepr='dodgy', offset=262, start_offset=262, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=264, start_offset=264, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=274, start_offset=274, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=286, start_offset=286, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=288, start_offset=288, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=290, start_offset=290, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=3, argval=3, argrepr='', offset=292, start_offset=292, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=302, start_offset=302, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=312, start_offset=312, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=322, start_offset=322, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=324, start_offset=324, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=43, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=348, argrepr='to L11', offset=340, start_offset=340, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=352, start_offset=352, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=354, start_offset=354, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=29, argval=302, argrepr='to L10', offset=358, start_offset=358, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=360, start_offset=360, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=368, start_offset=368, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=15, argval=414, argrepr='to L12', offset=380, start_offset=380, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=388, start_offset=388, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=398, start_offset=398, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=400, start_offset=400, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=408, start_offset=408, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=410, start_offset=410, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=56, argval=302, argrepr='to L10', offset=412, start_offset=412, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=414, start_offset=414, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=416, start_offset=416, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=418, start_offset=418, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=424, start_offset=424, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=434, start_offset=434, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=436, start_offset=436, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=444, start_offset=444, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=446, start_offset=446, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=448, start_offset=448, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=450, start_offset=450, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=452, start_offset=452, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), - Instruction(opname='RETURN_VALUE', opcode=34, arg=None, argval=None, argrepr='', offset=4, start_offset=4, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=4, start_offset=4, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), ] diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 32b3a6ac049e28..43e3e56639db62 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1589,11 +1589,11 @@ def whilefunc(n=0): ('branch right', 'whilefunc', 1, 3)]) self.check_events(func, recorders = BRANCH_OFFSET_RECORDERS, expected = [ - ('branch left', 'func', 28, 34), - ('branch right', 'func', 46, 60), - ('branch left', 'func', 28, 34), - ('branch left', 'func', 46, 52), - ('branch right', 'func', 28, 72)]) + ('branch left', 'func', 28, 32), + ('branch right', 'func', 44, 58), + ('branch left', 'func', 28, 32), + ('branch left', 'func', 44, 50), + ('branch right', 'func', 28, 70)]) def test_except_star(self): @@ -1658,6 +1658,88 @@ def foo(n=0): exit_loop]) +class TestBranchConsistency(MonitoringTestBase, unittest.TestCase): + + def check_branches(self, func, tool=TEST_TOOL, recorders=BRANCH_OFFSET_RECORDERS): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_local_events(tool, func.__code__, all_events) + func() + sys.monitoring.set_local_events(tool, func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + lefts = set() + rights = set() + for (src, left, right) in func.__code__.co_branches(): + lefts.add((src, left)) + rights.add((src, right)) + for event in event_list: + way, _, src, dest = event + if "left" in way: + self.assertIn((src, dest), lefts) + else: + self.assertIn("right", way) + self.assertIn((src, dest), rights) + finally: + sys.monitoring.set_local_events(tool, func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + def test_simple(self): + + def func(): + x = 1 + for a in range(2): + if a: + x = 4 + else: + x = 6 + 7 + + self.check_branches(func) + + def whilefunc(n=0): + while n < 3: + n += 1 # line 2 + 3 + + self.check_branches(whilefunc) + + def test_except_star(self): + + class Foo: + def meth(self): + pass + + def func(): + try: + try: + raise KeyError + except* Exception as e: + f = Foo(); f.meth() + except KeyError: + pass + + + self.check_branches(func) + + def test4(self): + + def foo(n=0): + while n<4: + pass + n += 1 + return None + + self.check_branches(foo) + + class TestLoadSuperAttr(CheckEvents): RECORDERS = CallRecorder, LineRecorder, CRaiseRecorder, CReturnRecorder diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index cd9014118f2d7f..a0a1f8af6710a3 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3415,6 +3415,26 @@ test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +static PyObject* +code_offset_to_line(PyObject* self, PyObject* const* args, Py_ssize_t nargsf) +{ + Py_ssize_t nargs = _PyVectorcall_NARGS(nargsf); + if (nargs != 2) { + PyErr_SetString(PyExc_TypeError, "code_offset_to_line takes 2 arguments"); + return NULL; + } + int offset; + if (PyLong_AsInt32(args[1], &offset) < 0) { + return NULL; + } + PyCodeObject *code = (PyCodeObject *)args[0]; + if (!PyCode_Check(code)) { + PyErr_SetString(PyExc_TypeError, "first arg must be a code object"); + return NULL; + } + return PyLong_FromInt32(PyCode_Addr2Line(code, offset)); +} + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -3557,6 +3577,7 @@ static PyMethodDef TestMethods[] = { {"finalize_thread_hang", finalize_thread_hang, METH_O, NULL}, {"type_freeze", type_freeze, METH_VARARGS}, {"test_atexit", test_atexit, METH_NOARGS}, + {"code_offset_to_line", _PyCFunction_CAST(code_offset_to_line), METH_FASTCALL}, {NULL, NULL} /* sentinel */ }; diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index a0007830e8cbc0..4f6933ac0ddcd6 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,37 +1,37 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, - 0,0,0,0,0,243,170,0,0,0,149,0,90,0,80,0, - 71,0,112,0,90,0,80,0,71,1,112,1,89,2,32,0, - 80,1,50,1,0,0,0,0,0,0,30,0,89,2,32,0, - 80,2,89,0,78,6,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,50,2,0,0,0,0,0,0, - 30,0,89,1,78,8,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,32,0,50,0,0,0,0,0, - 0,0,80,3,2,0,0,0,112,5,80,4,16,0,68,21, - 0,0,28,0,112,6,89,2,32,0,80,5,89,6,12,0, - 80,6,89,5,89,6,2,0,0,0,12,0,48,4,50,1, - 0,0,0,0,0,0,30,0,73,23,0,0,9,0,30,0, - 80,0,34,0,41,7,78,122,18,70,114,111,122,101,110,32, - 72,101,108,108,111,32,87,111,114,108,100,122,8,115,121,115, - 46,97,114,103,118,218,6,99,111,110,102,105,103,41,5,218, - 12,112,114,111,103,114,97,109,95,110,97,109,101,218,10,101, - 120,101,99,117,116,97,98,108,101,218,15,117,115,101,95,101, - 110,118,105,114,111,110,109,101,110,116,218,17,99,111,110,102, - 105,103,117,114,101,95,99,95,115,116,100,105,111,218,14,98, - 117,102,102,101,114,101,100,95,115,116,100,105,111,122,7,99, - 111,110,102,105,103,32,122,2,58,32,41,7,218,3,115,121, - 115,218,17,95,116,101,115,116,105,110,116,101,114,110,97,108, - 99,97,112,105,218,5,112,114,105,110,116,218,4,97,114,103, - 118,218,11,103,101,116,95,99,111,110,102,105,103,115,114,2, - 0,0,0,218,3,107,101,121,169,0,243,0,0,0,0,218, - 18,116,101,115,116,95,102,114,111,122,101,110,109,97,105,110, - 46,112,121,218,8,60,109,111,100,117,108,101,62,114,17,0, - 0,0,1,0,0,0,115,94,0,0,0,240,3,1,1,1, - 243,8,0,1,11,219,0,24,225,0,5,208,6,26,212,0, - 27,217,0,5,128,106,144,35,151,40,145,40,212,0,27,216, - 9,26,215,9,38,210,9,38,211,9,40,168,24,209,9,50, - 128,6,244,2,6,12,2,128,67,241,14,0,5,10,136,71, - 144,67,144,53,152,2,152,54,160,35,153,59,152,45,208,10, - 40,214,4,41,243,15,6,12,2,114,15,0,0,0, + 0,0,0,0,0,243,168,0,0,0,149,0,91,0,81,0, + 72,0,113,0,91,0,81,0,72,1,113,1,90,2,33,0, + 81,1,51,1,0,0,0,0,0,0,31,0,90,2,33,0, + 81,2,90,0,79,6,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,51,2,0,0,0,0,0,0, + 31,0,90,1,79,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,33,0,51,0,0,0,0,0, + 0,0,81,3,2,0,0,0,113,5,81,4,16,0,69,20, + 0,0,113,6,90,2,33,0,81,5,90,6,12,0,81,6, + 90,5,90,6,2,0,0,0,12,0,49,4,51,1,0,0, + 0,0,0,0,31,0,74,22,0,0,9,0,30,0,81,0, + 35,0,41,7,78,122,18,70,114,111,122,101,110,32,72,101, + 108,108,111,32,87,111,114,108,100,122,8,115,121,115,46,97, + 114,103,118,218,6,99,111,110,102,105,103,41,5,218,12,112, + 114,111,103,114,97,109,95,110,97,109,101,218,10,101,120,101, + 99,117,116,97,98,108,101,218,15,117,115,101,95,101,110,118, + 105,114,111,110,109,101,110,116,218,17,99,111,110,102,105,103, + 117,114,101,95,99,95,115,116,100,105,111,218,14,98,117,102, + 102,101,114,101,100,95,115,116,100,105,111,122,7,99,111,110, + 102,105,103,32,122,2,58,32,41,7,218,3,115,121,115,218, + 17,95,116,101,115,116,105,110,116,101,114,110,97,108,99,97, + 112,105,218,5,112,114,105,110,116,218,4,97,114,103,118,218, + 11,103,101,116,95,99,111,110,102,105,103,115,114,2,0,0, + 0,218,3,107,101,121,169,0,243,0,0,0,0,218,18,116, + 101,115,116,95,102,114,111,122,101,110,109,97,105,110,46,112, + 121,218,8,60,109,111,100,117,108,101,62,114,17,0,0,0, + 1,0,0,0,115,94,0,0,0,240,3,1,1,1,243,8, + 0,1,11,219,0,24,225,0,5,208,6,26,212,0,27,217, + 0,5,128,106,144,35,151,40,145,40,212,0,27,216,9,26, + 215,9,38,210,9,38,211,9,40,168,24,209,9,50,128,6, + 243,2,6,12,2,128,67,241,14,0,5,10,136,71,144,67, + 144,53,152,2,152,54,160,35,153,59,152,45,208,10,40,214, + 4,41,243,15,6,12,2,114,15,0,0,0, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 602cf7f47b812b..4961693c7e654a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -60,6 +60,8 @@ #define specializing #define split #define replicate(TIMES) +#define tier1 +#define no_save_ip // Dummy variables for stack effects. static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; @@ -336,9 +338,18 @@ dummy_func( res = PyStackRef_NULL; } - macro(END_FOR) = POP_TOP; + no_save_ip inst(END_FOR, (value -- )) { + /* Don't update instr_ptr, so that POP_ITER sees + * the FOR_ITER as the previous instruction. + * This has the benign side effect that if value is + * finalized it will see the location as the FOR_ITER's. + */ + PyStackRef_CLOSE(value); + } + + macro(POP_ITER) = POP_TOP; - tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { + no_save_ip tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyStackRef_GenCheck(receiver)) { @@ -350,6 +361,11 @@ dummy_func( DECREF_INPUTS(); } + tier1 inst(INSTRUMENTED_POP_ITER, (iter -- )) { + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + PyStackRef_CLOSE(iter); + } + pure inst(END_SEND, (receiver, value -- val)) { (void)receiver; val = value; @@ -2924,10 +2940,8 @@ dummy_func( /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR */ + JUMPBY(oparg + 1); DISPATCH(); } next = PyStackRef_FromPyObjectSteal(next_o); @@ -2957,12 +2971,14 @@ dummy_func( macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; + inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { _PyStackRef iter_stackref = TOP(); PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next != NULL) { PUSH(PyStackRef_FromPyObjectSteal(next)); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { if (_PyErr_Occurred(tstate)) { @@ -2976,14 +2992,12 @@ dummy_func( /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - PyStackRef_CLOSE(iter_stackref); - /* Skip END_FOR and POP_TOP */ - _Py_CODEUNIT *target = next_instr + oparg + 2; - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); + /* Skip END_FOR */ + JUMPBY(oparg + 1); } } + op(_ITER_CHECK_LIST, (iter -- iter)) { EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type); } @@ -3002,10 +3016,8 @@ dummy_func( Py_DECREF(seq); } #endif - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); DISPATCH(); } } @@ -3054,10 +3066,8 @@ dummy_func( it->it_seq = NULL; Py_DECREF(seq); } - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); DISPATCH(); } } @@ -3098,10 +3108,8 @@ dummy_func( assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); if (r->len <= 0) { - STACK_SHRINK(1); - PyStackRef_CLOSE(iter); - // Jump over END_FOR and POP_TOP instructions. - JUMPBY(oparg + 2); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); DISPATCH(); } } @@ -4779,7 +4787,8 @@ dummy_func( } inst(INSTRUMENTED_NOT_TAKEN, ( -- )) { - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + (void)this_instr; // INSTRUMENTED_JUMP requires this_instr + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } macro(INSTRUMENTED_JUMP_BACKWARD) = diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f15633fa467376..c37e1cf3afa60e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -363,7 +363,7 @@ do { \ next_instr = dest; \ } else { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, dest); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ next_instr = (dest)+1; \ diff --git a/Python/codegen.c b/Python/codegen.c index 7432415b17414e..14f9f5ad1254f5 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -409,7 +409,7 @@ codegen_addop_j(instr_sequence *seq, location loc, if (_PyInstructionSequence_Addop(seq, opcode, target.id, loc) != SUCCESS) { return ERROR; } - if (IS_CONDITIONAL_JUMP_OPCODE(opcode) || opcode == FOR_ITER) { + if (IS_CONDITIONAL_JUMP_OPCODE(opcode)) { return _PyInstructionSequence_Addop(seq, NOT_TAKEN, 0, NO_LOCATION); } return SUCCESS; @@ -2018,7 +2018,7 @@ codegen_for(compiler *c, stmt_ty s) * but a non-generator will jump to a later instruction. */ ADDOP(c, NO_LOCATION, END_FOR); - ADDOP(c, NO_LOCATION, POP_TOP); + ADDOP(c, NO_LOCATION, POP_ITER); _PyCompile_PopFBlock(c, COMPILE_FBLOCK_FOR_LOOP, start); @@ -4283,7 +4283,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc, * but a non-generator will jump to a later instruction. */ ADDOP(c, NO_LOCATION, END_FOR); - ADDOP(c, NO_LOCATION, POP_TOP); + ADDOP(c, NO_LOCATION, POP_ITER); } return SUCCESS; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f7374d52705960..ac2f69b7e98dc3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -411,6 +411,20 @@ break; } + case _END_FOR: { + _PyStackRef value; + value = stack_pointer[-1]; + /* Don't update instr_ptr, so that POP_ITER sees + * the FOR_ITER as the previous instruction. + * This has the benign side effect that if value is + * finalized it will see the location as the FOR_ITER's. + */ + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _END_SEND: { _PyStackRef value; _PyStackRef receiver; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 98743c27c38524..b73844ca2d9542 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3768,11 +3768,15 @@ } TARGET(END_FOR) { - frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; value = stack_pointer[-1]; + /* Don't update instr_ptr, so that POP_ITER sees + * the FOR_ITER as the previous instruction. + * This has the benign side effect that if value is + * finalized it will see the location as the FOR_ITER's. + */ PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3957,10 +3961,8 @@ /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR */ + JUMPBY(oparg + 1); DISPATCH(); } next = PyStackRef_FromPyObjectSteal(next_o); @@ -4048,10 +4050,8 @@ Py_DECREF(seq); } #endif - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); DISPATCH(); } } @@ -4091,10 +4091,8 @@ assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); if (r->len <= 0) { - STACK_SHRINK(1); - PyStackRef_CLOSE(iter); - // Jump over END_FOR and POP_TOP instructions. - JUMPBY(oparg + 2); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); DISPATCH(); } } @@ -4141,10 +4139,8 @@ it->it_seq = NULL; Py_DECREF(seq); } - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); DISPATCH(); } } @@ -4572,7 +4568,7 @@ } TARGET(INSTRUMENTED_END_FOR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); @@ -4636,6 +4632,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (next != NULL) { PUSH(PyStackRef_FromPyObjectSteal(next)); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { if (_PyErr_Occurred(tstate)) { @@ -4653,11 +4650,8 @@ /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - PyStackRef_CLOSE(iter_stackref); - /* Skip END_FOR and POP_TOP */ - _Py_CODEUNIT *target = next_instr + oparg + 2; - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); + /* Skip END_FOR */ + JUMPBY(oparg + 1); } DISPATCH(); } @@ -4764,11 +4758,28 @@ } TARGET(INSTRUMENTED_NOT_TAKEN) { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + (void)this_instr; // INSTRUMENTED_JUMP requires this_instr + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_ITER) { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + _PyStackRef iter; + iter = stack_pointer[-1]; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + PyStackRef_CLOSE(iter); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6693,6 +6704,18 @@ DISPATCH(); } + TARGET(POP_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + TARGET(POP_JUMP_IF_FALSE) { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index e4255bfad8c41a..17e5346be5ed3d 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -14,6 +14,7 @@ #include "pycore_namespace.h" #include "pycore_object.h" #include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches +#include "pycore_opcode_utils.h" // IS_CONDITIONAL_JUMP_OPCODE #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_UINTPTR_RELEASE #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -95,8 +96,10 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT, [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, - [FOR_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT, - [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [FOR_ITER] = PY_MONITORING_EVENT_BRANCH_LEFT, + [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH_LEFT, + [POP_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT, [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, [INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, [END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, @@ -119,6 +122,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, [INSTRUMENTED_FOR_ITER] = FOR_ITER, + [INSTRUMENTED_POP_ITER] = POP_ITER, [INSTRUMENTED_END_FOR] = END_FOR, [INSTRUMENTED_END_SEND] = END_SEND, [INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, @@ -156,6 +160,8 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, [FOR_ITER] = INSTRUMENTED_FOR_ITER, [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, + [POP_ITER] = INSTRUMENTED_POP_ITER, + [INSTRUMENTED_POP_ITER] = INSTRUMENTED_POP_ITER, [LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, [NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, @@ -1077,8 +1083,8 @@ static const char *const event_names [] = { static int call_instrumentation_vector( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[]) + _Py_CODEUNIT *instr, PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *arg2, Py_ssize_t nargs, PyObject *args[]) { if (tstate->tracing) { return 0; @@ -1091,17 +1097,13 @@ call_instrumentation_vector( int offset = (int)(instr - _PyFrame_GetBytecode(frame)); /* Offset visible to user should be the offset in bytes, as that is the * convention for APIs involving code offsets. */ - int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); - if (event == PY_MONITORING_EVENT_BRANCH_LEFT) { - assert(EVENT_FOR_OPCODE[_Py_GetBaseCodeUnit(code, offset-2).op.code] == PY_MONITORING_EVENT_BRANCH_RIGHT); - bytes_offset -= 4; - } - PyObject *offset_obj = PyLong_FromLong(bytes_offset); - if (offset_obj == NULL) { + int bytes_arg2 = (int)(arg2 - _PyFrame_GetBytecode(frame)) * (int)sizeof(_Py_CODEUNIT); + PyObject *arg2_obj = PyLong_FromLong(bytes_arg2); + if (arg2_obj == NULL) { return -1; } assert(args[2] == NULL); - args[2] = offset_obj; + args[2] = arg2_obj; PyInterpreterState *interp = tstate->interp; uint8_t tools = get_tools_for_instruction(code, interp, offset, event); size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; @@ -1139,7 +1141,7 @@ call_instrumentation_vector( } } } - Py_DECREF(offset_obj); + Py_DECREF(arg2_obj); return err; } @@ -1149,7 +1151,7 @@ _Py_call_instrumentation( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { PyObject *args[3] = { NULL, NULL, NULL }; - return call_instrumentation_vector(tstate, event, frame, instr, 2, args); + return call_instrumentation_vector(instr, tstate, event, frame, instr, 2, args); } int @@ -1158,7 +1160,7 @@ _Py_call_instrumentation_arg( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { PyObject *args[4] = { NULL, NULL, NULL, arg }; - return call_instrumentation_vector(tstate, event, frame, instr, 3, args); + return call_instrumentation_vector(instr, tstate, event, frame, instr, 3, args); } int @@ -1167,34 +1169,34 @@ _Py_call_instrumentation_2args( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) { PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 }; - return call_instrumentation_vector(tstate, event, frame, instr, 4, args); + return call_instrumentation_vector(instr, tstate, event, frame, instr, 4, args); } _Py_CODEUNIT * _Py_call_instrumentation_jump( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target) + _Py_CODEUNIT *instr, PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest) { assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH_RIGHT || event == PY_MONITORING_EVENT_BRANCH_LEFT); - assert(frame->instr_ptr == instr); - int to = (int)(target - _PyFrame_GetBytecode(frame)); + int to = (int)(dest - _PyFrame_GetBytecode(frame)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); if (to_obj == NULL) { return NULL; } PyObject *args[4] = { NULL, NULL, NULL, to_obj }; - int err = call_instrumentation_vector(tstate, event, frame, instr, 3, args); + _Py_CODEUNIT *instr_ptr = frame->instr_ptr; + int err = call_instrumentation_vector(instr, tstate, event, frame, src, 3, args); Py_DECREF(to_obj); if (err) { return NULL; } - if (frame->instr_ptr != instr) { + if (frame->instr_ptr != instr_ptr) { /* The callback has caused a jump (by setting the line number) */ return frame->instr_ptr; } - return target; + return dest; } static void @@ -1204,7 +1206,7 @@ call_instrumentation_vector_protected( { assert(_PyErr_Occurred(tstate)); PyObject *exc = _PyErr_GetRaisedException(tstate); - int err = call_instrumentation_vector(tstate, event, frame, instr, nargs, args); + int err = call_instrumentation_vector(instr, tstate, event, frame, instr, nargs, args); if (err) { Py_XDECREF(exc); } @@ -1496,9 +1498,10 @@ initialize_lines(PyCodeObject *code) case END_FOR: case END_SEND: case RESUME: + case POP_ITER: /* END_FOR cannot start a line, as it is skipped by FOR_ITER * END_SEND cannot start a line, as it is skipped by SEND - * RESUME must not be instrumented with INSTRUMENT_LINE */ + * RESUME and POP_ITER must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; break; default: @@ -1570,11 +1573,14 @@ initialize_lines(PyCodeObject *code) } assert(target >= 0); if (line_data[target].line_delta != NO_LINE) { - line_data[target].original_opcode = _Py_GetBaseCodeUnit(code, target).op.code; - if (line_data[target].line_delta == COMPUTED_LINE_LINENO_CHANGE) { - // If the line is a jump target, we are not sure if the line - // number changes, so we set it to COMPUTED_LINE. - line_data[target].line_delta = COMPUTED_LINE; + int opcode = _Py_GetBaseCodeUnit(code, target).op.code; + if (opcode != POP_ITER) { + line_data[target].original_opcode = opcode; + if (line_data[target].line_delta == COMPUTED_LINE_LINENO_CHANGE) { + // If the line is a jump target, we are not sure if the line + // number changes, so we set it to COMPUTED_LINE. + line_data[target].line_delta = COMPUTED_LINE; + } } } } @@ -2887,30 +2893,52 @@ branch_handler( _PyLegacyBranchEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { + // Find the other instrumented instruction and remove tool + // The spec (PEP 669) allows spurious events after a DISABLE, + // so a best effort is good enough. + assert(PyVectorcall_NARGS(nargsf) >= 3); + PyCodeObject *code = (PyCodeObject *)args[0]; + int src_offset = PyLong_AsLong(args[1]); + if (PyErr_Occurred()) { + return NULL; + } + _Py_CODEUNIT instr = _PyCode_CODE(code)[src_offset/2]; + if (!is_instrumented(instr.op.code)) { + /* Already disabled */ + return &_PyInstrumentation_DISABLE; + } PyObject *res = PyObject_Vectorcall(self->handler, args, nargsf, kwnames); if (res == &_PyInstrumentation_DISABLE) { - // Find the other instrumented instruction and remove tool - assert(PyVectorcall_NARGS(nargsf) >= 2); - PyObject *offset_obj = args[1]; - int bytes_offset = PyLong_AsLong(offset_obj); - if (PyErr_Occurred()) { - return NULL; - } - PyCodeObject *code = (PyCodeObject *)args[0]; - if (!PyCode_Check(code) || (bytes_offset & 1)) { - return res; - } - int offset = bytes_offset / 2; /* We need FOR_ITER and POP_JUMP_ to be the same size */ assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1); - if (self->right) { - offset += 2; + int offset; + int other_event; + if (instr.op.code == FOR_ITER) { + if (self->right) { + offset = src_offset/2; + other_event = PY_MONITORING_EVENT_BRANCH_LEFT; + } + else { + // We don't know where the POP_ITER is, so + // we cannot de-instrument it. + return res; + } + } + else if (IS_CONDITIONAL_JUMP_OPCODE(instr.op.code)) { + if (self->right) { + offset = src_offset/2 + 2; + other_event = PY_MONITORING_EVENT_BRANCH_LEFT; + assert(_Py_GetBaseCodeUnit(code, offset).op.code == NOT_TAKEN); + } + else { + offset = src_offset/2; + other_event = PY_MONITORING_EVENT_BRANCH_RIGHT; + } } - if (offset >= Py_SIZE(code)) { + else { + // Orphaned NOT_TAKEN -- Jump removed by the compiler return res; } - int other_event = self->right ? - PY_MONITORING_EVENT_BRANCH_LEFT : PY_MONITORING_EVENT_BRANCH_RIGHT; LOCK_CODE(code); remove_tools(code, offset, other_event, 1 << self->tool_id); UNLOCK_CODE(); @@ -3013,15 +3041,30 @@ static PyObject * branchesiter_next(branchesiterator *bi) { int offset = bi->bi_offset; + int oparg = 0; while (offset < Py_SIZE(bi->bi_code)) { _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(bi->bi_code, offset); - int next_offset = offset + _PyInstruction_GetLength(bi->bi_code, offset); - int event = EVENT_FOR_OPCODE[inst.op.code]; - if (event == PY_MONITORING_EVENT_BRANCH_RIGHT) { - /* Skip NOT_TAKEN */ - int not_taken = next_offset + 1; - bi->bi_offset = not_taken; - return int_triple(offset*2, not_taken*2, (next_offset + inst.op.arg)*2); + int next_offset = offset + 1 + _PyOpcode_Caches[inst.op.code]; + switch(inst.op.code) { + case EXTENDED_ARG: + oparg = (oparg << 8) | inst.op.arg; + break; + case FOR_ITER: + oparg = (oparg << 8) | inst.op.arg; + bi->bi_offset = next_offset; + int target = next_offset + oparg+2; // Skips END_FOR and POP_ITER + return int_triple(offset*2, next_offset*2, target*2); + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + oparg = (oparg << 8) | inst.op.arg; + /* Skip NOT_TAKEN */ + int not_taken = next_offset + 1; + bi->bi_offset = not_taken; + return int_triple(offset*2, not_taken*2, (next_offset + oparg)*2); + default: + oparg = 0; } offset = next_offset; } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 7f3fb9c9a63dd1..00d55d3a82edc1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -29,6 +29,7 @@ static void *opcode_targets[256] = { &&TARGET_NOP, &&TARGET_NOT_TAKEN, &&TARGET_POP_EXCEPT, + &&TARGET_POP_ITER, &&TARGET_POP_TOP, &&TARGET_PUSH_EXC_INFO, &&TARGET_PUSH_NULL, @@ -147,7 +148,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -234,8 +234,8 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, + &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, &&TARGET_INSTRUMENTED_FOR_ITER, diff --git a/Python/optimizer.c b/Python/optimizer.c index 6a4d20fad76c15..52b3f0a84afedf 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -622,8 +622,14 @@ translate_bytecode_to_trace( goto done; } assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - RESERVE_RAW(2, "_CHECK_VALIDITY_AND_SET_IP"); - ADD_TO_TRACE(_CHECK_VALIDITY_AND_SET_IP, 0, (uintptr_t)instr, target); + if (OPCODE_HAS_NO_SAVE_IP(opcode)) { + RESERVE_RAW(2, "_CHECK_VALIDITY"); + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + } + else { + RESERVE_RAW(2, "_CHECK_VALIDITY_AND_SET_IP"); + ADD_TO_TRACE(_CHECK_VALIDITY_AND_SET_IP, 0, (uintptr_t)instr, target); + } /* Special case the first instruction, * so that we can guarantee forward progress */ @@ -771,7 +777,7 @@ translate_bytecode_to_trace( uint32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + (oparg > 255); uint32_t jump_target = next_inst + oparg; assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_TOP); + assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); } #endif break; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 0fcf5e18ed5808..be3e06108aec92 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -115,6 +115,12 @@ break; } + case _END_FOR: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _END_SEND: { _Py_UopsSymbol *val; val = sym_new_not_null(ctx); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 73c871759afbf5..c0a370a936aa94 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -27,6 +27,7 @@ class Properties: oparg_and_1: bool = False const_oparg: int = -1 needs_prev: bool = False + no_save_ip: bool = False def dump(self, indent: str) -> None: simple_properties = self.__dict__.copy() @@ -60,6 +61,7 @@ def from_list(properties: list["Properties"]) -> "Properties": side_exit=any(p.side_exit for p in properties), pure=all(p.pure for p in properties), needs_prev=any(p.needs_prev for p in properties), + no_save_ip=all(p.no_save_ip for p in properties), ) @property @@ -87,6 +89,7 @@ def escapes(self) -> bool: has_free=False, side_exit=False, pure=True, + no_save_ip=False, ) @@ -829,6 +832,7 @@ def compute_properties(op: parser.InstDef) -> Properties: and not has_free, has_free=has_free, pure="pure" in op.annotations, + no_save_ip="no_save_ip" in op.annotations, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index d17617cab0266b..8df9a9cada92ae 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -583,6 +583,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_ESCAPES_FLAG") if p.pure: flags.append("HAS_PURE_FLAG") + if p.no_save_ip: + flags.append("HAS_NO_SAVE_IP_FLAG") if p.oparg_and_1: flags.append("HAS_OPARG_AND_1_FLAG") if flags: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 37f96398ff175f..bee2a185745f4d 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -226,6 +226,7 @@ def choice(*opts: str) -> str: "replicate", "tier1", "tier2", + "no_save_ip", } __all__ = [] diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 1a9849c0cbbb25..453db6905d6842 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -53,6 +53,7 @@ "PASSTHROUGH", "OPARG_AND_1", "ERROR_NO_POP", + "NO_SAVE_IP", ] @@ -285,8 +286,8 @@ def generate_metadata_table(analysis: Analysis, out: CWriter) -> None: table_size = 256 + len(analysis.pseudos) out.emit("struct opcode_metadata {\n") out.emit("uint8_t valid_entry;\n") - out.emit("int8_t instr_format;\n") - out.emit("int16_t flags;\n") + out.emit("uint8_t instr_format;\n") + out.emit("uint16_t flags;\n") out.emit("};\n\n") out.emit( f"extern const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}];\n" diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index fcdd3bdacd0e7a..40562da99b20ea 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -151,9 +151,12 @@ def generate_tier1( if inst.properties.needs_prev: out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") if needs_this and not inst.is_target: - out.emit(f"_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;\n") + if inst.properties.no_save_ip: + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") + else: + out.emit(f"_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;\n") out.emit(unused_guard) - else: + elif not inst.properties.no_save_ip: out.emit(f"frame->instr_ptr = next_instr;\n") out.emit(f"next_instr += {inst.size};\n") out.emit(f"INSTRUCTION_STATS({name});\n") From b6c919b674549d519efbbc378588ca6d3efc7242 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 6 Jan 2025 20:45:14 +0100 Subject: [PATCH 387/427] gh-127146: Fix test_sysconfigdata_json for Emscripten (#128556) --- Lib/sysconfig/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index ed7b6a335d01d4..20d506bcd45abc 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -485,10 +485,10 @@ def _init_config_vars(): _init_posix(_CONFIG_VARS) # If we are cross-compiling, load the prefixes from the Makefile instead. if '_PYTHON_PROJECT_BASE' in os.environ: - prefix = _CONFIG_VARS['prefix'] - exec_prefix = _CONFIG_VARS['exec_prefix'] - base_prefix = _CONFIG_VARS['prefix'] - base_exec_prefix = _CONFIG_VARS['exec_prefix'] + prefix = _CONFIG_VARS['host_prefix'] + exec_prefix = _CONFIG_VARS['host_exec_prefix'] + base_prefix = _CONFIG_VARS['host_prefix'] + base_exec_prefix = _CONFIG_VARS['host_exec_prefix'] abiflags = _CONFIG_VARS['ABIFLAGS'] # Normalized versions of prefix and exec_prefix are handy to have; From 2434fd2d50b8b770585ad5949a664e4bbab4bde1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 6 Jan 2025 22:01:07 +0000 Subject: [PATCH 388/427] GH-128533: Add `NOT_TAKEN` instruction after bytecode optimization. (GH-128554) --- Lib/test/test_compiler_codegen.py | 1 - Lib/test/test_dis.py | 248 +++++++++++++++--------------- Python/codegen.c | 8 +- Python/flowgraph.c | 9 +- 4 files changed, 127 insertions(+), 139 deletions(-) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 5655e1b9cf196a..cf5e2d901db4de 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -29,7 +29,6 @@ def test_if_expression(self): ('LOAD_CONST', 0, 1), ('TO_BOOL', 0, 1), ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1), - ('NOT_TAKEN', None, 1), ('LOAD_SMALL_INT', 42, 1), ('JUMP_NO_INTERRUPT', exit_lbl := self.Label()), false_lbl, diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index e733a673d003e7..76d9b5401d7d8e 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -200,8 +200,7 @@ def bug1333982(x=[]): dis_bug1333982 = """\ %3d RESUME 0 -%3d NOT_TAKEN - LOAD_COMMON_CONSTANT 0 (AssertionError) +%3d LOAD_COMMON_CONSTANT 0 (AssertionError) LOAD_CONST 0 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION LOAD_FAST 0 (x) @@ -433,7 +432,7 @@ def foo(a: int, b: str) -> str: 1 LOAD_SMALL_INT 0 STORE_NAME 0 (x) - 2 L1: NOT_TAKEN + 2 L1: NOP 3 LOAD_NAME 0 (x) LOAD_SMALL_INT 1 @@ -648,11 +647,11 @@ async def _asyncwith(c): L20: CLEANUP_THROW L21: END_SEND TO_BOOL - POP_JUMP_IF_TRUE 2 (to L22) - NOT_TAKEN - RERAISE 2 - L22: POP_TOP - L23: POP_EXCEPT + POP_JUMP_IF_TRUE 2 (to L24) + L22: NOT_TAKEN + L23: RERAISE 2 + L24: POP_TOP + L25: POP_EXCEPT POP_TOP POP_TOP POP_TOP @@ -662,24 +661,25 @@ async def _asyncwith(c): LOAD_CONST 0 (None) RETURN_VALUE - -- L24: COPY 3 + -- L26: COPY 3 POP_EXCEPT RERAISE 1 - L25: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + L27: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) RERAISE 1 ExceptionTable: - L1 to L3 -> L25 [0] lasti + L1 to L3 -> L27 [0] lasti L3 to L4 -> L12 [4] - L4 to L6 -> L25 [0] lasti + L4 to L6 -> L27 [0] lasti L6 to L7 -> L16 [2] lasti - L7 to L9 -> L25 [0] lasti + L7 to L9 -> L27 [0] lasti L9 to L10 -> L14 [2] - L10 to L13 -> L25 [0] lasti - L14 to L15 -> L25 [0] lasti - L16 to L18 -> L24 [4] lasti + L10 to L13 -> L27 [0] lasti + L14 to L15 -> L27 [0] lasti + L16 to L18 -> L26 [4] lasti L18 to L19 -> L20 [7] - L19 to L23 -> L24 [4] lasti - L23 to L25 -> L25 [0] lasti + L19 to L22 -> L26 [4] lasti + L23 to L25 -> L26 [4] lasti + L25 to L27 -> L27 [0] lasti """ % (_asyncwith.__code__.co_firstlineno, _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 2, @@ -1777,7 +1777,7 @@ def _prepare_test_cases(): Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='GET_ITER', opcode=16, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='FOR_ITER', opcode=69, arg=33, argval=94, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='FOR_ITER', opcode=69, arg=32, argval=92, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), @@ -1795,113 +1795,111 @@ def _prepare_test_cases(): Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=3, argval=88, argrepr='to L3', offset=78, start_offset=78, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=82, start_offset=82, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), Instruction(opname='JUMP_BACKWARD', opcode=74, arg=32, argval=24, argrepr='to L1', offset=84, start_offset=84, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=None, label=3, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_FORWARD', opcode=76, arg=13, argval=120, argrepr='to L5', offset=92, start_offset=92, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), - Instruction(opname='POP_ITER', opcode=30, arg=None, argval=None, argrepr='', offset=96, start_offset=96, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=98, start_offset=98, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=108, start_offset=108, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=118, start_offset=118, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=85, arg=0, argval='i', argrepr='i', offset=120, start_offset=120, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=122, start_offset=122, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=37, argval=208, argrepr='to L8', offset=130, start_offset=130, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=134, start_offset=134, starts_line=False, line_number=11, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=136, start_offset=136, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=146, start_offset=146, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=156, start_offset=156, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=160, start_offset=160, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=44, arg=23, argval=23, argrepr='-=', offset=162, start_offset=162, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=168, start_offset=168, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=6, argval=6, argrepr='', offset=170, start_offset=170, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=56, arg=148, argval='>', argrepr='bool(>)', offset=172, start_offset=172, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=3, argval=186, argrepr='to L6', offset=176, start_offset=176, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD', opcode=74, arg=33, argval=120, argrepr='to L5', offset=182, start_offset=182, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=186, start_offset=186, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=4, argval=4, argrepr='', offset=188, start_offset=188, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=56, arg=18, argval='<', argrepr='bool(<)', offset=190, start_offset=190, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=3, argval=204, argrepr='to L7', offset=194, start_offset=194, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=198, start_offset=198, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD', opcode=74, arg=42, argval=120, argrepr='to L5', offset=200, start_offset=200, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=204, start_offset=204, starts_line=True, line_number=None, label=7, positions=None, cache_info=None), - Instruction(opname='JUMP_FORWARD', opcode=76, arg=11, argval=230, argrepr='to L9', offset=206, start_offset=206, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=208, start_offset=208, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=232, start_offset=232, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=0, argval=0, argrepr='', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=44, arg=11, argval=11, argrepr='/', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=242, start_offset=242, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=59, arg=1, argval=1, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=92, arg=1, argval=1, argrepr='__exit__', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=114, arg=2, argval=2, argrepr='', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=114, arg=3, argval=3, argrepr='', offset=250, start_offset=250, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=92, arg=0, argval=0, argrepr='__enter__', offset=252, start_offset=252, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=0, argval=0, argrepr='', offset=254, start_offset=254, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=109, arg=1, argval='dodgy', argrepr='dodgy', offset=262, start_offset=262, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=264, start_offset=264, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=274, start_offset=274, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=286, start_offset=286, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=288, start_offset=288, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=290, start_offset=290, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=3, argval=3, argrepr='', offset=292, start_offset=292, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=302, start_offset=302, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=312, start_offset=312, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=322, start_offset=322, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=324, start_offset=324, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='WITH_EXCEPT_START', opcode=43, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=348, argrepr='to L11', offset=340, start_offset=340, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=76, arg=13, argval=118, argrepr='to L5', offset=90, start_offset=90, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=92, start_offset=92, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), + Instruction(opname='POP_ITER', opcode=30, arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=96, start_offset=96, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=106, start_offset=106, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=108, start_offset=108, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=85, arg=0, argval='i', argrepr='i', offset=118, start_offset=118, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=120, start_offset=120, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=36, argval=204, argrepr='to L8', offset=128, start_offset=128, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=132, start_offset=132, starts_line=False, line_number=11, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=134, start_offset=134, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=144, start_offset=144, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=146, start_offset=146, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=154, start_offset=154, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=156, start_offset=156, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=44, arg=23, argval=23, argrepr='-=', offset=160, start_offset=160, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=164, start_offset=164, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=6, argval=6, argrepr='', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=148, argval='>', argrepr='bool(>)', offset=170, start_offset=170, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=3, argval=184, argrepr='to L6', offset=174, start_offset=174, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=178, start_offset=178, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=33, argval=118, argrepr='to L5', offset=180, start_offset=180, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=184, start_offset=184, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=4, argval=4, argrepr='', offset=186, start_offset=186, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=18, argval='<', argrepr='bool(<)', offset=188, start_offset=188, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=3, argval=202, argrepr='to L7', offset=192, start_offset=192, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=196, start_offset=196, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=42, argval=118, argrepr='to L5', offset=198, start_offset=198, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_FORWARD', opcode=76, arg=11, argval=226, argrepr='to L9', offset=202, start_offset=202, starts_line=True, line_number=17, label=7, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=204, start_offset=204, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=214, start_offset=214, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=216, start_offset=216, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=224, start_offset=224, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=228, start_offset=228, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=0, argval=0, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=44, arg=11, argval=11, argrepr='/', offset=232, start_offset=232, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=238, start_offset=238, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=1, argval=1, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=92, arg=1, argval=1, argrepr='__exit__', offset=242, start_offset=242, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=114, arg=2, argval=2, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=114, arg=3, argval=3, argrepr='', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=92, arg=0, argval=0, argrepr='__enter__', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=0, argval=0, argrepr='', offset=250, start_offset=250, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=1, argval='dodgy', argrepr='dodgy', offset=258, start_offset=258, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=260, start_offset=260, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=270, start_offset=270, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=272, start_offset=272, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=280, start_offset=280, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=282, start_offset=282, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=284, start_offset=284, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=286, start_offset=286, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=3, argval=3, argrepr='', offset=288, start_offset=288, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=298, start_offset=298, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=308, start_offset=308, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=310, start_offset=310, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=320, start_offset=320, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=322, start_offset=322, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=324, start_offset=324, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=43, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=344, argrepr='to L11', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=352, start_offset=352, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=354, start_offset=354, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=29, argval=302, argrepr='to L10', offset=358, start_offset=358, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=360, start_offset=360, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=368, start_offset=368, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=15, argval=414, argrepr='to L12', offset=380, start_offset=380, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=388, start_offset=388, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=398, start_offset=398, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=400, start_offset=400, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=408, start_offset=408, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=410, start_offset=410, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=56, argval=302, argrepr='to L10', offset=412, start_offset=412, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=414, start_offset=414, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=416, start_offset=416, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=418, start_offset=418, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=424, start_offset=424, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=434, start_offset=434, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=436, start_offset=436, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=444, start_offset=444, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=446, start_offset=446, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=448, start_offset=448, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=450, start_offset=450, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=452, start_offset=452, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=29, argval=298, argrepr='to L10', offset=354, start_offset=354, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=356, start_offset=356, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=364, start_offset=364, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=15, argval=410, argrepr='to L12', offset=376, start_offset=376, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=28, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=382, start_offset=382, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=384, start_offset=384, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=394, start_offset=394, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=404, start_offset=404, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=56, argval=298, argrepr='to L10', offset=408, start_offset=408, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=410, start_offset=410, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=412, start_offset=412, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=416, start_offset=416, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=418, start_offset=418, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=420, start_offset=420, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=430, start_offset=430, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=51, arg=1, argval=1, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=440, start_offset=440, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=442, start_offset=442, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=444, start_offset=444, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=29, arg=None, argval=None, argrepr='', offset=446, start_offset=446, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Python/codegen.c b/Python/codegen.c index 14f9f5ad1254f5..61707ba677097c 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -406,13 +406,7 @@ codegen_addop_j(instr_sequence *seq, location loc, assert(IS_JUMP_TARGET_LABEL(target)); assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - if (_PyInstructionSequence_Addop(seq, opcode, target.id, loc) != SUCCESS) { - return ERROR; - } - if (IS_CONDITIONAL_JUMP_OPCODE(opcode)) { - return _PyInstructionSequence_Addop(seq, NOT_TAKEN, 0, NO_LOCATION); - } - return SUCCESS; + return _PyInstructionSequence_Addop(seq, opcode, target.id, loc); } #define ADDOP_JUMP(C, LOC, OP, O) \ diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 64df6290de06ba..017216aadd1f01 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -522,14 +522,15 @@ no_redundant_jumps(cfg_builder *g) { static int normalize_jumps_in_block(cfg_builder *g, basicblock *b) { cfg_instr *last = basicblock_last_instr(b); - if (last == NULL || !is_jump(last) || - IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { + if (last == NULL || !IS_CONDITIONAL_JUMP_OPCODE(last->i_opcode)) { return SUCCESS; } assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); bool is_forward = last->i_target->b_visited == 0; if (is_forward) { + RETURN_IF_ERROR( + basicblock_addop(b, NOT_TAKEN, 0, last->i_loc)); return SUCCESS; } @@ -557,10 +558,6 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } - assert(b->b_next->b_iused > 0); - assert(b->b_next->b_instr[0].i_opcode == NOT_TAKEN); - b->b_next->b_instr[0].i_opcode = NOP; - b->b_next->b_instr[0].i_loc = NO_LOCATION; RETURN_IF_ERROR( basicblock_addop(backwards_jump, NOT_TAKEN, 0, last->i_loc)); RETURN_IF_ERROR( From 616692d465c4ef10baa592e41f3d9daeae4dddad Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 6 Jan 2025 23:25:29 +0100 Subject: [PATCH 389/427] gh-127146: Skip Emscripten tests with pending fixes (#128549) Marks some additional tests as skipped on Emscripten. Many of these skips can be reverted when the next Emscripten release is available. --- Lib/test/test_io.py | 1 + Lib/test/test_os.py | 1 + Lib/test/test_shutil.py | 1 + Lib/test/test_tarfile.py | 1 + Lib/test/test_zipfile/test_core.py | 4 +++- Lib/test/test_zipimport.py | 1 + 6 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 81c17b2731cc58..33e0161241e87e 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3933,6 +3933,7 @@ def test_issue35928(self): self.assertEqual(res + f.readline(), 'foo\nbar\n') @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @unittest.skipIf(support.is_emscripten, "Would be fixed by emscripten-core/emscripten#23306") def test_read_non_blocking(self): import os r, w = os.pipe() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index d688a225538c11..d2c4dff3c9a0e5 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4979,6 +4979,7 @@ def test_unpickable(self): self.assertRaises(TypeError, pickle.dumps, scandir_iter, filename) scandir_iter.close() + @unittest.skipIf(support.is_emscripten, "Fixed by emscripten-core/emscripten#23139, remove when next Emscripten release comes out") def check_entry(self, entry, name, is_dir, is_file, is_symlink): self.assertIsInstance(entry, os.DirEntry) self.assertEqual(entry.name, name) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 1f18b1f09b5858..2d79d2ffede461 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1587,6 +1587,7 @@ def test_copyfile_same_file(self): # the path as a directory, but on AIX the trailing slash has no effect # and is considered as a file. @unittest.skipIf(AIX, 'Not valid on AIX, see gh-92670') + @unittest.skipIf(support.is_emscripten, 'Fixed by emscripten-core/emscripten#23218, remove when next Emscripten release comes out') def test_copyfile_nonexistent_dir(self): # Issue 43219 src_dir = self.mkdtemp() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 54d329a15d4d25..2549b6b35adc29 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3800,6 +3800,7 @@ def test_absolute_hardlink(self): "'parent' is a link to an absolute path") @symlink_test + @unittest.skipIf(support.is_emscripten, "Fixed by emscripten-core/emscripten#23136, remove when next Emscripten release comes out") def test_sly_relative0(self): # Inspired by 'relative0' in jwilk/traversal-archives with ArchiveMaker() as arc: diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 49f39b9337df85..79e7337606b4bc 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -22,7 +22,8 @@ from test.support import script_helper from test.support import ( findfile, requires_zlib, requires_bz2, requires_lzma, - captured_stdout, captured_stderr, requires_subprocess + captured_stdout, captured_stderr, requires_subprocess, + is_emscripten ) from test.support.os_helper import ( TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count, FakePath @@ -622,6 +623,7 @@ def test_write_to_readonly(self): with self.assertRaises(ValueError): zipfp.open(TESTFN, mode='w') + @unittest.skipIf(is_emscripten, "Fixed by emscripten-core/emscripten#23310") def test_add_file_before_1980(self): # Set atime and mtime to 1970-01-01 os.utime(TESTFN, (0, 0)) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 1f288c8b45d589..65f8b17f2f88c0 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -1038,6 +1038,7 @@ def testEmptyFile(self): self.assertZipFailure(TESTMOD) @unittest.skipIf(support.is_wasi, "mode 000 not supported.") + @unittest.skipIf(support.is_emscripten, "Fixed by emscripten-core/emscripten#23137, remove when next Emscripten release comes out") def testFileUnreadable(self): os_helper.unlink(TESTMOD) fd = os.open(TESTMOD, os.O_CREAT, 000) From 61c1a2478e6da8dc6dbdce4d6ac66b03d5963710 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 6 Jan 2025 23:26:35 +0100 Subject: [PATCH 390/427] gh-127146: Strip dash from Emscripten compiler version (#128557) `emcc -dumpversion` will sometimes say e.g., `4.0.0-git` but in this case uname does not include `-git` in the version string. Use cut to delete everything after the dash. --- configure | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 61ee51c4b36473..9024d22d4df5cd 100755 --- a/configure +++ b/configure @@ -4595,7 +4595,7 @@ printf "%s\n" "$IPHONEOS_DEPLOYMENT_TARGET" >&6; } _host_ident=$host_cpu ;; *-*-emscripten) - _host_ident=$(emcc -dumpversion)-$host_cpu + _host_ident=$(emcc -dumpversion | cut -f1 -d-)-$host_cpu ;; wasm32-*-* | wasm64-*-*) _host_ident=$host_cpu diff --git a/configure.ac b/configure.ac index 172e8a1a842010..86775e03e73d5a 100644 --- a/configure.ac +++ b/configure.ac @@ -794,7 +794,7 @@ if test "$cross_compiling" = yes; then _host_ident=$host_cpu ;; *-*-emscripten) - _host_ident=$(emcc -dumpversion)-$host_cpu + _host_ident=$(emcc -dumpversion | cut -f1 -d-)-$host_cpu ;; wasm32-*-* | wasm64-*-*) _host_ident=$host_cpu From 7363476b6405e3d288a61282aa7bc6aca9c2114d Mon Sep 17 00:00:00 2001 From: Michael H Date: Mon, 6 Jan 2025 18:03:27 -0500 Subject: [PATCH 391/427] gh-128559: Remove typing import from asyncio.timeouts (#128560) --- Lib/asyncio/timeouts.py | 24 +++++++++---------- ...-01-06-21-35-00.gh-issue-128559.6fxcDM.rst | 1 + 2 files changed, 12 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-06-21-35-00.gh-issue-128559.6fxcDM.rst diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index e6f5100691d362..09342dc7c1310b 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -1,7 +1,6 @@ import enum from types import TracebackType -from typing import final, Optional, Type from . import events from . import exceptions @@ -23,14 +22,13 @@ class _State(enum.Enum): EXITED = "finished" -@final class Timeout: """Asynchronous context manager for cancelling overdue coroutines. Use `timeout()` or `timeout_at()` rather than instantiating this class directly. """ - def __init__(self, when: Optional[float]) -> None: + def __init__(self, when: float | None) -> None: """Schedule a timeout that will trigger at a given loop time. - If `when` is `None`, the timeout will never trigger. @@ -39,15 +37,15 @@ def __init__(self, when: Optional[float]) -> None: """ self._state = _State.CREATED - self._timeout_handler: Optional[events.TimerHandle] = None - self._task: Optional[tasks.Task] = None + self._timeout_handler: events.TimerHandle | None = None + self._task: tasks.Task | None = None self._when = when - def when(self) -> Optional[float]: + def when(self) -> float | None: """Return the current deadline.""" return self._when - def reschedule(self, when: Optional[float]) -> None: + def reschedule(self, when: float | None) -> None: """Reschedule the timeout.""" if self._state is not _State.ENTERED: if self._state is _State.CREATED: @@ -96,10 +94,10 @@ async def __aenter__(self) -> "Timeout": async def __aexit__( self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType], - ) -> Optional[bool]: + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool | None: assert self._state in (_State.ENTERED, _State.EXPIRING) if self._timeout_handler is not None: @@ -142,7 +140,7 @@ def _insert_timeout_error(exc_val: BaseException) -> None: exc_val = exc_val.__context__ -def timeout(delay: Optional[float]) -> Timeout: +def timeout(delay: float | None) -> Timeout: """Timeout async context manager. Useful in cases when you want to apply timeout logic around block @@ -162,7 +160,7 @@ def timeout(delay: Optional[float]) -> Timeout: return Timeout(loop.time() + delay if delay is not None else None) -def timeout_at(when: Optional[float]) -> Timeout: +def timeout_at(when: float | None) -> Timeout: """Schedule the timeout at absolute time. Like timeout() but argument gives absolute time in the same clock system diff --git a/Misc/NEWS.d/next/Library/2025-01-06-21-35-00.gh-issue-128559.6fxcDM.rst b/Misc/NEWS.d/next/Library/2025-01-06-21-35-00.gh-issue-128559.6fxcDM.rst new file mode 100644 index 00000000000000..7f9380de17761b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-06-21-35-00.gh-issue-128559.6fxcDM.rst @@ -0,0 +1 @@ +Improved import time of :mod:`asyncio`. From 953b49e5468d02afaddadc2307f4763422078603 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 6 Jan 2025 23:38:45 +0000 Subject: [PATCH 392/427] gh-108202: Document `calendar.Calendar` methods `getfirstweekday` and `setfirstweekday` (#127579) --- Doc/library/calendar.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index ace8529d6e7e0c..dadd301ebc795b 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -40,6 +40,14 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :class:`Calendar` instances have the following methods: + .. method:: getfirstweekday() + + Return an :class:`int` for the current first weekday (0-6). + + .. method:: setfirstweekday(firstweekday) + + Set the first weekday to *firstweekday*, passed as an :class:`int` where Monday is 0 and Sunday is 6. + .. method:: iterweekdays() Return an iterator for the week day numbers that will be used for one From 24b147a19b360c49cb1740aa46211d342aaa071f Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Mon, 6 Jan 2025 21:02:55 -0600 Subject: [PATCH 393/427] gh-128472: Add `-skip-funcs` to BOLT options to fix computed goto errors (gh-128511) * Add `-skip-funcs` to BOLT options to fix computed goto errors Co-authored-by: Gregory Szorc * NEWS --------- Co-authored-by: Gregory Szorc --- .../Build/2025-01-04-22-39-10.gh-issue-128472.Wt5E6M.rst | 2 ++ configure | 2 +- configure.ac | 9 ++++++++- 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-01-04-22-39-10.gh-issue-128472.Wt5E6M.rst diff --git a/Misc/NEWS.d/next/Build/2025-01-04-22-39-10.gh-issue-128472.Wt5E6M.rst b/Misc/NEWS.d/next/Build/2025-01-04-22-39-10.gh-issue-128472.Wt5E6M.rst new file mode 100644 index 00000000000000..c6233e1f2d8693 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-01-04-22-39-10.gh-issue-128472.Wt5E6M.rst @@ -0,0 +1,2 @@ +Skip BOLT optimization of functions using computed gotos, fixing errors on +build with LLVM 19. diff --git a/configure b/configure index 9024d22d4df5cd..08c10ff52ce8d5 100755 --- a/configure +++ b/configure @@ -9398,7 +9398,7 @@ fi printf %s "checking BOLT_COMMON_FLAGS... " >&6; } if test -z "${BOLT_COMMON_FLAGS}" then - BOLT_COMMON_FLAGS=-update-debug-sections + BOLT_COMMON_FLAGS=" -update-debug-sections -skip-funcs=_PyEval_EvalFrameDefault,sre_ucs1_match/1,sre_ucs2_match/1,sre_ucs4_match/1 " fi diff --git a/configure.ac b/configure.ac index 86775e03e73d5a..fb1dd77be167b9 100644 --- a/configure.ac +++ b/configure.ac @@ -2170,7 +2170,14 @@ if test -z "${BOLT_COMMON_FLAGS}" then AS_VAR_SET( [BOLT_COMMON_FLAGS], - [-update-debug-sections] + [m4_normalize(" + [-update-debug-sections] + + dnl At least LLVM 19.x doesn't support computed gotos in PIC compiled code. + dnl Exclude functions containing computed gotos. + dnl TODO this may be fixed in LLVM 20.x via https://github.com/llvm/llvm-project/pull/120267. + [-skip-funcs=_PyEval_EvalFrameDefault,sre_ucs1_match/1,sre_ucs2_match/1,sre_ucs4_match/1] + ")] ) fi From e837a1f71e832ce8f551a6fac05e346f654457e0 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 7 Jan 2025 13:12:58 +0800 Subject: [PATCH 394/427] gh-128146: Exclude os/log.h import on older macOS versions. (#128165) Reworks the handling of Apple system log handling to account for older macOS versions that don't provide os-log. --- ...-12-22-08-54-30.gh-issue-127592.iyuFCC.rst | 2 ++ Python/pylifecycle.c | 33 ++++++++++++------- 2 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst diff --git a/Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst b/Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst new file mode 100644 index 00000000000000..dfe659294c712e --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst @@ -0,0 +1,2 @@ +Usage of the unified Apple System Log APIs was disabled when the minimum +macOS version is earlier than 10.12. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 06418123d6dd9b..8a15a09a01dbf7 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -46,8 +46,25 @@ #if defined(__APPLE__) # include +# include # include -# include +// The os_log unified logging APIs were introduced in macOS 10.12, iOS 10.0, +// tvOS 10.0, and watchOS 3.0; +# if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE +# define HAS_APPLE_SYSTEM_LOG 1 +# elif defined(TARGET_OS_OSX) && TARGET_OS_OSX +# if defined(MAC_OS_X_VERSION_10_12) && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 +# define HAS_APPLE_SYSTEM_LOG 1 +# else +# define HAS_APPLE_SYSTEM_LOG 0 +# endif +# else +# define HAS_APPLE_SYSTEM_LOG 0 +# endif + +# if HAS_APPLE_SYSTEM_LOG +# include +# endif #endif #ifdef HAVE_SIGNAL_H @@ -77,7 +94,7 @@ static PyStatus init_sys_streams(PyThreadState *tstate); #ifdef __ANDROID__ static PyStatus init_android_streams(PyThreadState *tstate); #endif -#if defined(__APPLE__) +#if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG static PyStatus init_apple_streams(PyThreadState *tstate); #endif static void wait_for_thread_shutdown(PyThreadState *tstate); @@ -1262,7 +1279,7 @@ init_interp_main(PyThreadState *tstate) return status; } #endif -#if defined(__APPLE__) +#if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG if (config->use_system_logger) { status = init_apple_streams(tstate); if (_PyStatus_EXCEPTION(status)) { @@ -2946,7 +2963,7 @@ init_android_streams(PyThreadState *tstate) #endif // __ANDROID__ -#if defined(__APPLE__) +#if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG static PyObject * apple_log_write_impl(PyObject *self, PyObject *args) @@ -2957,14 +2974,9 @@ apple_log_write_impl(PyObject *self, PyObject *args) return NULL; } - // Call the underlying Apple logging API. The os_log unified logging APIs - // were introduced in macOS 10.12, iOS 10.0, tvOS 10.0, and watchOS 3.0; - // this call is a no-op on older versions. - #if TARGET_OS_IPHONE || (TARGET_OS_OSX && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12) // Pass the user-provided text through explicit %s formatting // to avoid % literals being interpreted as a formatting directive. os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); - #endif Py_RETURN_NONE; } @@ -2999,7 +3011,6 @@ init_apple_streams(PyThreadState *tstate) if (result == NULL) { goto error; } - goto done; error: @@ -3013,7 +3024,7 @@ init_apple_streams(PyThreadState *tstate) return status; } -#endif // __APPLE__ +#endif // __APPLE__ && HAS_APPLE_SYSTEM_LOG static void From b3cbd8f1b58b9e71df20bdda9960e7d3190c509e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 7 Jan 2025 08:59:24 +0100 Subject: [PATCH 395/427] gh-108202: Document calendar.Calendar.firstweekday (#128566) --- Doc/library/calendar.rst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index dadd301ebc795b..f25930eb0d873e 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -38,21 +38,33 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is itself. This is the job of subclasses. - :class:`Calendar` instances have the following methods: + :class:`Calendar` instances have the following methods and attributes: + + .. attribute:: firstweekday + + The first weekday as an integer (0--6). + + This property can also be set and read using + :meth:`~Calendar.setfirstweekday` and + :meth:`~Calendar.getfirstweekday` respectively. .. method:: getfirstweekday() - Return an :class:`int` for the current first weekday (0-6). + Return an :class:`int` for the current first weekday (0--6). + + Identical to reading the :attr:`~Calendar.firstweekday` property. .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` where Monday is 0 and Sunday is 6. + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + + Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() Return an iterator for the week day numbers that will be used for one week. The first value from the iterator will be the same as the value of - the :attr:`firstweekday` property. + the :attr:`~Calendar.firstweekday` property. .. method:: itermonthdates(year, month) From 145276a072ae058bac42ee43a4235cd4eda2726b Mon Sep 17 00:00:00 2001 From: Jun Komoda <45822440+junkmd@users.noreply.github.com> Date: Tue, 7 Jan 2025 17:36:27 +0900 Subject: [PATCH 396/427] Add `.. availability:: Windows` directive to COM-related function prototype (GH-127436) Add a directive to prototype that "returns a foreign function that will call a COM method" --- Doc/library/ctypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 398cb92bac809a..f25bf94417c198 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1812,6 +1812,8 @@ different ways, depending on the type and number of the parameters in the call: the COM interface as first argument, in addition to those parameters that are specified in the :attr:`!argtypes` tuple. + .. availability:: Windows + The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. From 6ea04da27036eaa69d65150148bb8c537d9beacf Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Tue, 7 Jan 2025 02:40:41 -0800 Subject: [PATCH 397/427] gh-128302: Fix bugs in xml.dom.xmlbuilder (GH-128284) * Allow DOMParser.parse() to correctly handle DOMInputSource instances that only have a systemId attribute set. * Fix DOMEntityResolver.resolveEntity(), which was broken by the Python 3.0 transition. * Add Lib/test/test_xml_dom_xmlbuilder.py with few tests. --- Lib/test/test_xml_dom_xmlbuilder.py | 88 +++++++++++++++++++ Lib/xml/dom/xmlbuilder.py | 12 +-- ...-12-27-16-28-57.gh-issue-128302.2GMvyl.rst | 3 + ...-12-29-13-49-46.gh-issue-128302.psRpPN.rst | 2 + 4 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 Lib/test/test_xml_dom_xmlbuilder.py create mode 100644 Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst create mode 100644 Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst diff --git a/Lib/test/test_xml_dom_xmlbuilder.py b/Lib/test/test_xml_dom_xmlbuilder.py new file mode 100644 index 00000000000000..5f5f2eb328df9f --- /dev/null +++ b/Lib/test/test_xml_dom_xmlbuilder.py @@ -0,0 +1,88 @@ +import io +import unittest +from http import client +from test.test_httplib import FakeSocket +from unittest import mock +from xml.dom import getDOMImplementation, minidom, xmlbuilder + +SMALL_SAMPLE = b""" + + +Introduction to XSL +
+

A. Namespace

+""" + + +class XMLBuilderTest(unittest.TestCase): + def test_entity_resolver(self): + body = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" + + SMALL_SAMPLE + ) + + sock = FakeSocket(body) + response = client.HTTPResponse(sock) + response.begin() + attrs = {"open.return_value": response} + opener = mock.Mock(**attrs) + + resolver = xmlbuilder.DOMEntityResolver() + + with mock.patch("urllib.request.build_opener") as mock_build: + mock_build.return_value = opener + source = resolver.resolveEntity(None, "http://example.com/2000/svg") + + self.assertIsInstance(source, xmlbuilder.DOMInputSource) + self.assertIsNone(source.publicId) + self.assertEqual(source.systemId, "http://example.com/2000/svg") + self.assertEqual(source.baseURI, "http://example.com/2000/") + self.assertEqual(source.encoding, "utf-8") + self.assertIs(source.byteStream, response) + + self.assertIsNone(source.characterStream) + self.assertIsNone(source.stringData) + + def test_builder(self): + imp = getDOMImplementation() + self.assertIsInstance(imp, xmlbuilder.DOMImplementationLS) + + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + self.assertIsInstance(builder, xmlbuilder.DOMBuilder) + + def test_parse_uri(self): + body = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" + + SMALL_SAMPLE + ) + + sock = FakeSocket(body) + response = client.HTTPResponse(sock) + response.begin() + attrs = {"open.return_value": response} + opener = mock.Mock(**attrs) + + with mock.patch("urllib.request.build_opener") as mock_build: + mock_build.return_value = opener + + imp = getDOMImplementation() + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + document = builder.parseURI("http://example.com/2000/svg") + + self.assertIsInstance(document, minidom.Document) + self.assertEqual(len(document.childNodes), 1) + + def test_parse_with_systemId(self): + response = io.BytesIO(SMALL_SAMPLE) + + with mock.patch("urllib.request.urlopen") as mock_open: + mock_open.return_value = response + + imp = getDOMImplementation() + source = imp.createDOMInputSource() + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + source.systemId = "http://example.com/2000/svg" + document = builder.parse(source) + + self.assertIsInstance(document, minidom.Document) + self.assertEqual(len(document.childNodes), 1) diff --git a/Lib/xml/dom/xmlbuilder.py b/Lib/xml/dom/xmlbuilder.py index 8a200263497b89..a8852625a2f9a2 100644 --- a/Lib/xml/dom/xmlbuilder.py +++ b/Lib/xml/dom/xmlbuilder.py @@ -189,7 +189,7 @@ def parse(self, input): options.filter = self.filter options.errorHandler = self.errorHandler fp = input.byteStream - if fp is None and options.systemId: + if fp is None and input.systemId: import urllib.request fp = urllib.request.urlopen(input.systemId) return self._parse_bytestream(fp, options) @@ -247,10 +247,12 @@ def _create_opener(self): def _guess_media_encoding(self, source): info = source.byteStream.info() - if "Content-Type" in info: - for param in info.getplist(): - if param.startswith("charset="): - return param.split("=", 1)[1].lower() + # import email.message + # assert isinstance(info, email.message.Message) + charset = info.get_param('charset') + if charset is not None: + return charset.lower() + return None class DOMInputSource(object): diff --git a/Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst b/Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst new file mode 100644 index 00000000000000..56e2fe6f85f4bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst @@ -0,0 +1,3 @@ +Allow :meth:`!xml.dom.xmlbuilder.DOMParser.parse` to correctly handle +:class:`!xml.dom.xmlbuilder.DOMInputSource` instances that only have a +:attr:`!systemId` attribute set. diff --git a/Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst b/Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst new file mode 100644 index 00000000000000..98c07297b06f8a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst @@ -0,0 +1,2 @@ +Fix :meth:`!xml.dom.xmlbuilder.DOMEntityResolver.resolveEntity`, which was +broken by the Python 3.0 transition. From 61b9811ac6843e22b5896ef96030d421b79cd892 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 11:44:57 +0000 Subject: [PATCH 398/427] gh-128552: fix refcycles in eager task creation (#128553) --- Lib/asyncio/base_events.py | 7 ++- Lib/asyncio/taskgroups.py | 7 ++- Lib/test/test_asyncio/test_taskgroups.py | 62 +++++++++++++++++-- ...-01-06-18-41-08.gh-issue-128552.fV-f8j.rst | 1 + 4 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 9e6f6e3ee7e3ec..6e6e5aaac15caf 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -477,7 +477,12 @@ def create_task(self, coro, *, name=None, context=None): task.set_name(name) - return task + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->BaseEventLoop.create_task->task + del task def set_task_factory(self, factory): """Set a task factory that will be used by loop.create_task(). diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 9fa772ca9d02cc..8af199d6dcc41a 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -205,7 +205,12 @@ def create_task(self, coro, *, name=None, context=None): else: self._tasks.add(task) task.add_done_callback(self._on_task_done) - return task + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->TaskGroup.create_task->task + del task # Since Python 3.8 Tasks propagate all exceptions correctly, # except for KeyboardInterrupt and SystemExit which are diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index c47bf4ec9ed64b..870fa8dbbf2714 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -1,6 +1,7 @@ # Adapted with permission from the EdgeDB project; # license: PSFL. +import weakref import sys import gc import asyncio @@ -38,7 +39,25 @@ def no_other_refs(): return [coro] -class TestTaskGroup(unittest.IsolatedAsyncioTestCase): +def set_gc_state(enabled): + was_enabled = gc.isenabled() + if enabled: + gc.enable() + else: + gc.disable() + return was_enabled + + +@contextlib.contextmanager +def disable_gc(): + was_enabled = set_gc_state(enabled=False) + try: + yield + finally: + set_gc_state(enabled=was_enabled) + + +class BaseTestTaskGroup: async def test_taskgroup_01(self): @@ -832,15 +851,15 @@ async def test_taskgroup_without_parent_task(self): with self.assertRaisesRegex(RuntimeError, "has not been entered"): tg.create_task(coro) - def test_coro_closed_when_tg_closed(self): + async def test_coro_closed_when_tg_closed(self): async def run_coro_after_tg_closes(): async with taskgroups.TaskGroup() as tg: pass coro = asyncio.sleep(0) with self.assertRaisesRegex(RuntimeError, "is finished"): tg.create_task(coro) - loop = asyncio.get_event_loop() - loop.run_until_complete(run_coro_after_tg_closes()) + + await run_coro_after_tg_closes() async def test_cancelling_level_preserved(self): async def raise_after(t, e): @@ -965,6 +984,30 @@ async def coro_fn(): self.assertIsInstance(exc, _Done) self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_parent_task_wr(self): + """Test that TaskGroup deletes self._parent_task and create_task() deletes task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + with disable_gc(): + try: + async with asyncio.TaskGroup() as tg2: + task_wr = weakref.ref(tg2.create_task(coro_fn())) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsNone(task_wr()) + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + async def test_exception_refcycles_propagate_cancellation_error(self): """Test that TaskGroup deletes propagate_cancellation_error""" tg = asyncio.TaskGroup() @@ -998,5 +1041,16 @@ class MyKeyboardInterrupt(KeyboardInterrupt): self.assertListEqual(gc.get_referrers(exc), no_other_refs()) +class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + loop_factory = asyncio.EventLoop + +class TestEagerTaskTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + @staticmethod + def loop_factory(): + loop = asyncio.EventLoop() + loop.set_task_factory(asyncio.eager_task_factory) + return loop + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst b/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst new file mode 100644 index 00000000000000..83816f775da9c5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst @@ -0,0 +1 @@ +Fix cyclic garbage introduced by :meth:`asyncio.loop.create_task` and :meth:`asyncio.TaskGroup.create_task` holding a reference to the created task if it is eager. From bcdf654c8a9d8a661e32e6a89f043197de19ad53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:45:29 +0100 Subject: [PATCH 399/427] gh-111178: fix UBSan failures in `Modules/_abc.c` (GH-128253) * fix UBSan failures in `_abc.c` * suppress unused return values --- Modules/_abc.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 4f4b24b035db4a..d6a953b336025d 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -67,6 +67,8 @@ typedef struct { uint64_t _abc_negative_cache_version; } _abc_data; +#define _abc_data_CAST(op) ((_abc_data *)(op)) + static inline uint64_t get_cache_version(_abc_data *impl) { @@ -88,8 +90,9 @@ set_cache_version(_abc_data *impl, uint64_t version) } static int -abc_data_traverse(_abc_data *self, visitproc visit, void *arg) +abc_data_traverse(PyObject *op, visitproc visit, void *arg) { + _abc_data *self = _abc_data_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->_abc_registry); Py_VISIT(self->_abc_cache); @@ -98,8 +101,9 @@ abc_data_traverse(_abc_data *self, visitproc visit, void *arg) } static int -abc_data_clear(_abc_data *self) +abc_data_clear(PyObject *op) { + _abc_data *self = _abc_data_CAST(op); Py_CLEAR(self->_abc_registry); Py_CLEAR(self->_abc_cache); Py_CLEAR(self->_abc_negative_cache); @@ -107,7 +111,7 @@ abc_data_clear(_abc_data *self) } static void -abc_data_dealloc(_abc_data *self) +abc_data_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); PyTypeObject *tp = Py_TYPE(self); @@ -212,7 +216,7 @@ _destroy(PyObject *setweakref, PyObject *objweakref) } static PyMethodDef _destroy_def = { - "_destroy", (PyCFunction) _destroy, METH_O + "_destroy", _destroy, METH_O }; static int @@ -964,7 +968,7 @@ _abcmodule_clear(PyObject *module) static void _abcmodule_free(void *module) { - _abcmodule_clear((PyObject *)module); + (void)_abcmodule_clear((PyObject *)module); } static PyModuleDef_Slot _abcmodule_slots[] = { From a734c1e304ab459fb34a88f6dd2dbd944a1b57c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:47:18 +0100 Subject: [PATCH 400/427] gh-111178: fix UBSan failures in `Modules/_sre/sre.c` (GH-128250) fix UBSan failures for `PatternObject`, `MatchObject`, `TemplateObject`, `ScannerObject` --- Modules/_sre/sre.c | 91 ++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 36f542ddb4df2b..d0025dd21e045b 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -395,6 +395,11 @@ static struct PyModuleDef sremodule; static PyObject*pattern_new_match(_sremodulestate *, PatternObject*, SRE_STATE*, Py_ssize_t); static PyObject *pattern_scanner(_sremodulestate *, PatternObject *, PyObject *, Py_ssize_t, Py_ssize_t); +#define _PatternObject_CAST(op) ((PatternObject *)(op)) +#define _MatchObject_CAST(op) ((MatchObject *)(op)) +#define _TemplateObject_CAST(op) ((TemplateObject *)(op)) +#define _ScannerObject_CAST(op) ((ScannerObject *)(op)) + /*[clinic input] module _sre class _sre.SRE_Pattern "PatternObject *" "get_sre_module_state_by_class(tp)->Pattern_Type" @@ -699,8 +704,9 @@ pattern_error(Py_ssize_t status) } static int -pattern_traverse(PatternObject *self, visitproc visit, void *arg) +pattern_traverse(PyObject *op, visitproc visit, void *arg) { + PatternObject *self = _PatternObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->groupindex); Py_VISIT(self->indexgroup); @@ -712,8 +718,9 @@ pattern_traverse(PatternObject *self, visitproc visit, void *arg) } static int -pattern_clear(PatternObject *self) +pattern_clear(PyObject *op) { + PatternObject *self = _PatternObject_CAST(op); Py_CLEAR(self->groupindex); Py_CLEAR(self->indexgroup); Py_CLEAR(self->pattern); @@ -724,13 +731,13 @@ pattern_clear(PatternObject *self) } static void -pattern_dealloc(PatternObject* self) +pattern_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) self); + PatternObject *obj = _PatternObject_CAST(self); + if (obj->weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } (void)pattern_clear(self); tp->tp_free(self); @@ -1497,7 +1504,7 @@ _sre_SRE_Pattern__fail_after_impl(PatternObject *self, int count, #endif /* Py_DEBUG */ static PyObject * -pattern_repr(PatternObject *obj) +pattern_repr(PyObject *self) { static const struct { const char *name; @@ -1512,6 +1519,8 @@ pattern_repr(PatternObject *obj) {"re.DEBUG", SRE_FLAG_DEBUG}, {"re.ASCII", SRE_FLAG_ASCII}, }; + + PatternObject *obj = _PatternObject_CAST(self); PyObject *result = NULL; PyObject *flag_items; size_t i; @@ -1579,8 +1588,9 @@ PyDoc_STRVAR(pattern_doc, "Compiled regular expression object."); /* PatternObject's 'groupindex' method. */ static PyObject * -pattern_groupindex(PatternObject *self, void *Py_UNUSED(ignored)) +pattern_groupindex(PyObject *op, void *Py_UNUSED(ignored)) { + PatternObject *self = _PatternObject_CAST(op); if (self->groupindex == NULL) return PyDict_New(); return PyDictProxy_New(self->groupindex); @@ -2245,8 +2255,9 @@ _validate(PatternObject *self) /* match methods */ static int -match_traverse(MatchObject *self, visitproc visit, void *arg) +match_traverse(PyObject *op, visitproc visit, void *arg) { + MatchObject *self = _MatchObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->string); Py_VISIT(self->regs); @@ -2255,8 +2266,9 @@ match_traverse(MatchObject *self, visitproc visit, void *arg) } static int -match_clear(MatchObject *self) +match_clear(PyObject *op) { + MatchObject *self = _MatchObject_CAST(op); Py_CLEAR(self->string); Py_CLEAR(self->regs); Py_CLEAR(self->pattern); @@ -2264,10 +2276,9 @@ match_clear(MatchObject *self) } static void -match_dealloc(MatchObject* self) +match_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); (void)match_clear(self); tp->tp_free(self); @@ -2376,8 +2387,9 @@ _sre_SRE_Match_expand_impl(MatchObject *self, PyObject *template) } static PyObject* -match_group(MatchObject* self, PyObject* args) +match_group(PyObject *op, PyObject* args) { + MatchObject *self = _MatchObject_CAST(op); PyObject* result; Py_ssize_t i, size; @@ -2411,8 +2423,9 @@ match_group(MatchObject* self, PyObject* args) } static PyObject* -match_getitem(MatchObject* self, PyObject* name) +match_getitem(PyObject *op, PyObject* name) { + MatchObject *self = _MatchObject_CAST(op); return match_getslice(self, name, Py_None); } @@ -2654,16 +2667,18 @@ PyDoc_STRVAR(match_group_doc, For 0 returns the entire match."); static PyObject * -match_lastindex_get(MatchObject *self, void *Py_UNUSED(ignored)) +match_lastindex_get(PyObject *op, void *Py_UNUSED(ignored)) { + MatchObject *self = _MatchObject_CAST(op); if (self->lastindex >= 0) return PyLong_FromSsize_t(self->lastindex); Py_RETURN_NONE; } static PyObject * -match_lastgroup_get(MatchObject *self, void *Py_UNUSED(ignored)) +match_lastgroup_get(PyObject *op, void *Py_UNUSED(ignored)) { + MatchObject *self = _MatchObject_CAST(op); if (self->pattern->indexgroup && self->lastindex >= 0 && self->lastindex < PyTuple_GET_SIZE(self->pattern->indexgroup)) @@ -2676,8 +2691,9 @@ match_lastgroup_get(MatchObject *self, void *Py_UNUSED(ignored)) } static PyObject * -match_regs_get(MatchObject *self, void *Py_UNUSED(ignored)) +match_regs_get(PyObject *op, void *Py_UNUSED(ignored)) { + MatchObject *self = _MatchObject_CAST(op); if (self->regs) { return Py_NewRef(self->regs); } else @@ -2780,27 +2796,29 @@ pattern_new_match(_sremodulestate* module_state, /* scanner methods (experimental) */ static int -scanner_traverse(ScannerObject *self, visitproc visit, void *arg) +scanner_traverse(PyObject *op, visitproc visit, void *arg) { + ScannerObject *self = _ScannerObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->pattern); return 0; } static int -scanner_clear(ScannerObject *self) +scanner_clear(PyObject *op) { + ScannerObject *self = _ScannerObject_CAST(op); Py_CLEAR(self->pattern); return 0; } static void -scanner_dealloc(ScannerObject* self) +scanner_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - state_fini(&self->state); + ScannerObject *scanner = _ScannerObject_CAST(self); + state_fini(&scanner->state); (void)scanner_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -2957,8 +2975,9 @@ pattern_scanner(_sremodulestate *module_state, /* template methods */ static int -template_traverse(TemplateObject *self, visitproc visit, void *arg) +template_traverse(PyObject *op, visitproc visit, void *arg) { + TemplateObject *self = _TemplateObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->literal); for (Py_ssize_t i = 0, n = Py_SIZE(self); i < n; i++) { @@ -2968,8 +2987,9 @@ template_traverse(TemplateObject *self, visitproc visit, void *arg) } static int -template_clear(TemplateObject *self) +template_clear(PyObject *op) { + TemplateObject *self = _TemplateObject_CAST(op); Py_CLEAR(self->literal); for (Py_ssize_t i = 0, n = Py_SIZE(self); i < n; i++) { Py_CLEAR(self->items[i].literal); @@ -2978,10 +2998,9 @@ template_clear(TemplateObject *self) } static void -template_dealloc(TemplateObject *self) +template_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); (void)template_clear(self); tp->tp_free(self); @@ -3056,8 +3075,10 @@ expand_template(TemplateObject *self, MatchObject *match) static Py_hash_t -pattern_hash(PatternObject *self) +pattern_hash(PyObject *op) { + PatternObject *self = _PatternObject_CAST(op); + Py_hash_t hash, hash2; hash = PyObject_Hash(self->pattern); @@ -3148,7 +3169,7 @@ static PyMethodDef pattern_methods[] = { }; static PyGetSetDef pattern_getset[] = { - {"groupindex", (getter)pattern_groupindex, (setter)NULL, + {"groupindex", pattern_groupindex, NULL, "A dictionary mapping group names to group numbers."}, {NULL} /* Sentinel */ }; @@ -3166,9 +3187,9 @@ static PyMemberDef pattern_members[] = { }; static PyType_Slot pattern_slots[] = { - {Py_tp_dealloc, (destructor)pattern_dealloc}, - {Py_tp_repr, (reprfunc)pattern_repr}, - {Py_tp_hash, (hashfunc)pattern_hash}, + {Py_tp_dealloc, pattern_dealloc}, + {Py_tp_repr, pattern_repr}, + {Py_tp_hash, pattern_hash}, {Py_tp_doc, (void *)pattern_doc}, {Py_tp_richcompare, pattern_richcompare}, {Py_tp_methods, pattern_methods}, @@ -3189,7 +3210,7 @@ static PyType_Spec pattern_spec = { }; static PyMethodDef match_methods[] = { - {"group", (PyCFunction) match_group, METH_VARARGS, match_group_doc}, + {"group", match_group, METH_VARARGS, match_group_doc}, _SRE_SRE_MATCH_START_METHODDEF _SRE_SRE_MATCH_END_METHODDEF _SRE_SRE_MATCH_SPAN_METHODDEF @@ -3204,11 +3225,11 @@ static PyMethodDef match_methods[] = { }; static PyGetSetDef match_getset[] = { - {"lastindex", (getter)match_lastindex_get, (setter)NULL, + {"lastindex", match_lastindex_get, NULL, "The integer index of the last matched capturing group."}, - {"lastgroup", (getter)match_lastgroup_get, (setter)NULL, + {"lastgroup", match_lastgroup_get, NULL, "The name of the last matched capturing group."}, - {"regs", (getter)match_regs_get, (setter)NULL}, + {"regs", match_regs_get, NULL, NULL}, {NULL} }; From 8f93dd8a8f237b277abad20d566df90c5cbd7f1e Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Tue, 7 Jan 2025 06:41:01 -0800 Subject: [PATCH 401/427] gh-115999: Add free-threaded specialization for COMPARE_OP (#126410) Add free-threaded specialization for COMPARE_OP, and tests for COMPARE_OP specialization in general. Co-authored-by: Donghee Na --- Lib/test/test_opcache.py | 33 +++++++++++++++++++++++++++++++++ Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- Python/specialize.c | 17 +++++++---------- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 79f452f8068c7f..c7cd4c2e8a3146 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1598,6 +1598,39 @@ def __getitem__(self, item): self.assert_specialized(binary_subscr_getitems, "BINARY_SUBSCR_GETITEM") self.assert_no_opcode(binary_subscr_getitems, "BINARY_SUBSCR") + @cpython_only + @requires_specialization_ft + def test_compare_op(self): + def compare_op_int(): + for _ in range(100): + a, b = 1, 2 + c = a == b + self.assertFalse(c) + + compare_op_int() + self.assert_specialized(compare_op_int, "COMPARE_OP_INT") + self.assert_no_opcode(compare_op_int, "COMPARE_OP") + + def compare_op_float(): + for _ in range(100): + a, b = 1.0, 2.0 + c = a == b + self.assertFalse(c) + + compare_op_float() + self.assert_specialized(compare_op_float, "COMPARE_OP_FLOAT") + self.assert_no_opcode(compare_op_float, "COMPARE_OP") + + def compare_op_str(): + for _ in range(100): + a, b = "spam", "ham" + c = a == b + self.assertFalse(c) + + compare_op_str() + self.assert_specialized(compare_op_str, "COMPARE_OP_STR") + self.assert_no_opcode(compare_op_str, "COMPARE_OP") + if __name__ == "__main__": unittest.main() diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4961693c7e654a..ec1cd00962ac0a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2464,7 +2464,7 @@ dummy_func( }; specializing op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b73844ca2d9542..eaa8a563464068 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3229,7 +3229,7 @@ left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/specialize.c b/Python/specialize.c index c918c77779d20d..c9325c39210874 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2480,23 +2480,23 @@ _Py_Specialize_CompareOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *i { PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st); PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st); + uint8_t specialized_op; - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); // All of these specializations compute boolean values, so they're all valid // regardless of the fifth-lowest oparg bit. - _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); if (Py_TYPE(lhs) != Py_TYPE(rhs)) { SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); goto failure; } if (PyFloat_CheckExact(lhs)) { - instr->op.code = COMPARE_OP_FLOAT; + specialized_op = COMPARE_OP_FLOAT; goto success; } if (PyLong_CheckExact(lhs)) { if (_PyLong_IsCompact((PyLongObject *)lhs) && _PyLong_IsCompact((PyLongObject *)rhs)) { - instr->op.code = COMPARE_OP_INT; + specialized_op = COMPARE_OP_INT; goto success; } else { @@ -2511,19 +2511,16 @@ _Py_Specialize_CompareOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *i goto failure; } else { - instr->op.code = COMPARE_OP_STR; + specialized_op = COMPARE_OP_STR; goto success; } } SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); failure: - STAT_INC(COMPARE_OP, failure); - instr->op.code = COMPARE_OP; - cache->counter = adaptive_counter_backoff(cache->counter); + unspecialize(instr); return; success: - STAT_INC(COMPARE_OP, success); - cache->counter = adaptive_counter_cooldown(); + specialize(instr, specialized_op); } #ifdef Py_STATS From a21e31ec5417374497b0f969ee2b3dc84a0944ff Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 7 Jan 2025 22:07:17 +0100 Subject: [PATCH 402/427] Docs: mark up json.load() using parameter list (#128488) --- Doc/library/json.rst | 113 ++++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 44 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 169291f74f44a5..cf516cf3dc1d29 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -258,36 +258,82 @@ Basic Usage the original one. That is, ``loads(dumps(x)) != x`` if x has non-string keys. -.. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) +.. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, \ + parse_int=None, parse_constant=None, \ + object_pairs_hook=None, **kw) - Deserialize *fp* (a ``.read()``-supporting :term:`text file` or - :term:`binary file` containing a JSON document) to a Python object using - this :ref:`conversion table `. + Deserialize *fp* to a Python object + using the :ref:`JSON-to-Python conversion table `. - *object_hook* is an optional function that will be called with the result of - any object literal decoded (a :class:`dict`). The return value of - *object_hook* will be used instead of the :class:`dict`. This feature can - be used to implement custom decoders (e.g. `JSON-RPC - `_ class hinting). + :param fp: + A ``.read()``-supporting :term:`text file` or :term:`binary file` + containing the JSON document to be deserialized. + :type fp: :term:`file-like object` - *object_pairs_hook* is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders. If - *object_hook* is also defined, the *object_pairs_hook* takes priority. + :param cls: + If set, a custom JSON decoder. + Additional keyword arguments to :func:`!load` + will be passed to the constructor of *cls*. + If ``None`` (the default), :class:`!JSONDecoder` is used. + :type cls: a :class:`JSONDecoder` subclass + + :param object_hook: + If set, a function that is called with the result of + any object literal decoded (a :class:`dict`). + The return value of this function will be used + instead of the :class:`dict`. + This feature can be used to implement custom decoders, + for example `JSON-RPC `_ class hinting. + Default ``None``. + :type object_hook: :term:`callable` | None + + :param object_pairs_hook: + If set, a function that is called with the result of + any object literal decoded with an ordered list of pairs. + The return value of this function will be used + instead of the :class:`dict`. + This feature can be used to implement custom decoders. + If *object_hook* is also set, *object_pairs_hook* takes priority. + Default ``None``. + :type object_pairs_hook: :term:`callable` | None + + :param parse_float: + If set, a function that is called with + the string of every JSON float to be decoded. + If ``None`` (the default), it is equivalent to ``float(num_str)``. + This can be used to parse JSON floats into custom datatypes, + for example :class:`decimal.Decimal`. + :type parse_float: :term:`callable` | None + + :param parse_int: + If set, a function that is called with + the string of every JSON int to be decoded. + If ``None`` (the default), it is equivalent to ``int(num_str)``. + This can be used to parse JSON integers into custom datatypes, + for example :class:`float`. + :type parse_int: :term:`callable` | None + + :param parse_constant: + If set, a function that is called with one of the following strings: + ``'-Infinity'``, ``'Infinity'``, or ``'NaN'``. + This can be used to raise an exception + if invalid JSON numbers are encountered. + Default ``None``. + :type parse_constant: :term:`callable` | None + + :raises JSONDecodeError: + When the data being deserialized is not a valid JSON document. .. versionchanged:: 3.1 - Added support for *object_pairs_hook*. - *parse_float* is an optional function that will be called with the string of - every JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser for - JSON floats (e.g. :class:`decimal.Decimal`). + * Added the optional *object_pairs_hook* parameter. + * *parse_constant* doesn't get called on 'null', 'true', 'false' anymore. - *parse_int* is an optional function that will be called with the string of - every JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser for - JSON integers (e.g. :class:`float`). + .. versionchanged:: 3.6 + + * All optional parameters are now :ref:`keyword-only `. + * *fp* can now be a :term:`binary file`. + The input encoding should be UTF-8, UTF-16 or UTF-32. .. versionchanged:: 3.11 The default *parse_int* of :func:`int` now limits the maximum length of @@ -295,27 +341,6 @@ Basic Usage conversion length limitation ` to help avoid denial of service attacks. - *parse_constant* is an optional function that will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This can be - used to raise an exception if invalid JSON numbers are encountered. - - .. versionchanged:: 3.1 - *parse_constant* doesn't get called on 'null', 'true', 'false' anymore. - - To use a custom :class:`JSONDecoder` subclass, specify it with the ``cls`` - kwarg; otherwise :class:`JSONDecoder` is used. Additional keyword arguments - will be passed to the constructor of the class. - - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. - - .. versionchanged:: 3.6 - All optional parameters are now :ref:`keyword-only `. - - .. versionchanged:: 3.6 - *fp* can now be a :term:`binary file`. The input encoding should be - UTF-8, UTF-16 or UTF-32. - .. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) Deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` From 07e6aa2efc5b2c7ecbbdb41636013c64e335a2ba Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 7 Jan 2025 23:13:08 +0200 Subject: [PATCH 403/427] gh-127350: Add more tests for Py_fopen() (GH-128587) --- Lib/test/test_capi/test_file.py | 25 +++++++++++++++++++++---- Modules/_testcapi/clinic/file.c.h | 25 +++++++------------------ Modules/_testcapi/file.c | 8 +++++--- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_capi/test_file.py b/Lib/test/test_capi/test_file.py index 8a08a0a93eb8b7..a67a5121c4588b 100644 --- a/Lib/test/test_capi/test_file.py +++ b/Lib/test/test_capi/test_file.py @@ -5,6 +5,8 @@ _testcapi = import_helper.import_module('_testcapi') +NULL = None + class CAPIFileTest(unittest.TestCase): def test_py_fopen(self): @@ -25,7 +27,9 @@ def test_py_fopen(self): os_helper.TESTFN, os.fsencode(os_helper.TESTFN), ] - # TESTFN_UNDECODABLE cannot be used to create a file on macOS/WASI. + if os_helper.TESTFN_UNDECODABLE is not None: + filenames.append(os_helper.TESTFN_UNDECODABLE) + filenames.append(os.fsdecode(os_helper.TESTFN_UNDECODABLE)) if os_helper.TESTFN_UNENCODABLE is not None: filenames.append(os_helper.TESTFN_UNENCODABLE) for filename in filenames: @@ -33,7 +37,12 @@ def test_py_fopen(self): try: with open(filename, "wb") as fp: fp.write(source) - + except OSError: + # TESTFN_UNDECODABLE cannot be used to create a file + # on macOS/WASI. + filename = None + continue + try: data = _testcapi.py_fopen(filename, "rb") self.assertEqual(data, source[:256]) finally: @@ -47,7 +56,14 @@ def test_py_fopen(self): # non-ASCII mode failing with "Invalid argument" with self.assertRaises(OSError): - _testcapi.py_fopen(__file__, "\xe9") + _testcapi.py_fopen(__file__, b"\xc2\x80") + with self.assertRaises(OSError): + # \x98 is invalid in cp1250, cp1251, cp1257 + # \x9d is invalid in cp1252-cp1255, cp1258 + _testcapi.py_fopen(__file__, b"\xc2\x98\xc2\x9d") + # UnicodeDecodeError can come from the audit hook code + with self.assertRaises((UnicodeDecodeError, OSError)): + _testcapi.py_fopen(__file__, b"\x98\x9d") # invalid filename type for invalid_type in (123, object()): @@ -60,7 +76,8 @@ def test_py_fopen(self): # On Windows, the file mode is limited to 10 characters _testcapi.py_fopen(__file__, "rt+, ccs=UTF-8") - # CRASHES py_fopen(__file__, None) + # CRASHES _testcapi.py_fopen(NULL, 'rb') + # CRASHES _testcapi.py_fopen(__file__, NULL) if __name__ == "__main__": diff --git a/Modules/_testcapi/clinic/file.c.h b/Modules/_testcapi/clinic/file.c.h index 2ca21fffcef680..fddbf48071bd3b 100644 --- a/Modules/_testcapi/clinic/file.c.h +++ b/Modules/_testcapi/clinic/file.c.h @@ -14,7 +14,8 @@ PyDoc_STRVAR(_testcapi_py_fopen__doc__, {"py_fopen", _PyCFunction_CAST(_testcapi_py_fopen), METH_FASTCALL, _testcapi_py_fopen__doc__}, static PyObject * -_testcapi_py_fopen_impl(PyObject *module, PyObject *path, const char *mode); +_testcapi_py_fopen_impl(PyObject *module, PyObject *path, const char *mode, + Py_ssize_t mode_length); static PyObject * _testcapi_py_fopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -22,27 +23,15 @@ _testcapi_py_fopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *path; const char *mode; - - if (!_PyArg_CheckPositional("py_fopen", nargs, 2, 2)) { - goto exit; - } - path = args[0]; - if (!PyUnicode_Check(args[1])) { - _PyArg_BadArgument("py_fopen", "argument 2", "str", args[1]); - goto exit; - } Py_ssize_t mode_length; - mode = PyUnicode_AsUTF8AndSize(args[1], &mode_length); - if (mode == NULL) { - goto exit; - } - if (strlen(mode) != (size_t)mode_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); + + if (!_PyArg_ParseStack(args, nargs, "Oz#:py_fopen", + &path, &mode, &mode_length)) { goto exit; } - return_value = _testcapi_py_fopen_impl(module, path, mode); + return_value = _testcapi_py_fopen_impl(module, path, mode, mode_length); exit: return return_value; } -/*[clinic end generated code: output=c9fe964c3e5a0c32 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c4dc92400306c3eb input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/file.c b/Modules/_testcapi/file.c index 4bad43010fd440..d15173fc7959e5 100644 --- a/Modules/_testcapi/file.c +++ b/Modules/_testcapi/file.c @@ -14,16 +14,18 @@ module _testcapi _testcapi.py_fopen path: object - mode: str + mode: str(zeroes=True, accept={robuffer, str, NoneType}) / Call Py_fopen(), fread(256) and Py_fclose(). Return read bytes. [clinic start generated code]*/ static PyObject * -_testcapi_py_fopen_impl(PyObject *module, PyObject *path, const char *mode) -/*[clinic end generated code: output=5a900af000f759de input=d7e7b8f0fd151953]*/ +_testcapi_py_fopen_impl(PyObject *module, PyObject *path, const char *mode, + Py_ssize_t mode_length) +/*[clinic end generated code: output=69840d0cfd8b7fbb input=f3a579dd7eb60926]*/ { + NULLABLE(path); FILE *fp = Py_fopen(path, mode); if (fp == NULL) { return NULL; From e08b28235a863323ca3a7e444776bb7803e77caf Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 7 Jan 2025 22:42:36 +0100 Subject: [PATCH 404/427] gh-127614: Correctly check for ttyname_r() in configure (#128503) PR #14868 replaced the ttyname() call with ttyname_r(), but the old check remained. --- Modules/clinic/posixmodule.c.h | 6 +++--- Modules/posixmodule.c | 2 +- configure | 6 +++--- configure.ac | 2 +- pyconfig.h.in | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 554299b8598299..4e6c5b068c42b7 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -309,7 +309,7 @@ os_access(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } -#if defined(HAVE_TTYNAME) +#if defined(HAVE_TTYNAME_R) PyDoc_STRVAR(os_ttyname__doc__, "ttyname($module, fd, /)\n" @@ -342,7 +342,7 @@ os_ttyname(PyObject *module, PyObject *arg) return return_value; } -#endif /* defined(HAVE_TTYNAME) */ +#endif /* defined(HAVE_TTYNAME_R) */ #if defined(HAVE_CTERMID) @@ -13140,4 +13140,4 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__EMSCRIPTEN_DEBUGGER_METHODDEF #define OS__EMSCRIPTEN_DEBUGGER_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_DEBUGGER_METHODDEF) */ -/*[clinic end generated code: output=9c2ca1dbf986c62c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=39b69b279fd637f7 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 151d469983fafb..bb8d698bfed375 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3347,7 +3347,7 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, #endif -#ifdef HAVE_TTYNAME +#ifdef HAVE_TTYNAME_R /*[clinic input] os.ttyname diff --git a/configure b/configure index 08c10ff52ce8d5..6e1b393a3ece68 100755 --- a/configure +++ b/configure @@ -19986,10 +19986,10 @@ then : printf "%s\n" "#define HAVE_TRUNCATE 1" >>confdefs.h fi -ac_fn_c_check_func "$LINENO" "ttyname" "ac_cv_func_ttyname" -if test "x$ac_cv_func_ttyname" = xyes +ac_fn_c_check_func "$LINENO" "ttyname_r" "ac_cv_func_ttyname_r" +if test "x$ac_cv_func_ttyname_r" = xyes then : - printf "%s\n" "#define HAVE_TTYNAME 1" >>confdefs.h + printf "%s\n" "#define HAVE_TTYNAME_R 1" >>confdefs.h fi ac_fn_c_check_func "$LINENO" "umask" "ac_cv_func_umask" diff --git a/configure.ac b/configure.ac index fb1dd77be167b9..6d44be8959865d 100644 --- a/configure.ac +++ b/configure.ac @@ -5152,7 +5152,7 @@ AC_CHECK_FUNCS([ \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ - tmpnam tmpnam_r truncate ttyname umask uname unlinkat unlockpt utimensat utimes vfork \ + tmpnam tmpnam_r truncate ttyname_r umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) diff --git a/pyconfig.h.in b/pyconfig.h.in index d862966b7de4c8..874b98dc96585a 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1506,8 +1506,8 @@ /* Define to 1 if you have the 'truncate' function. */ #undef HAVE_TRUNCATE -/* Define to 1 if you have the 'ttyname' function. */ -#undef HAVE_TTYNAME +/* Define to 1 if you have the 'ttyname_r' function. */ +#undef HAVE_TTYNAME_R /* Define to 1 if you don't have 'tm_zone' but do have the external array 'tzname'. */ From 65ae3d5a73ca3c53a0c6b601dddb8e9b3b6e3f51 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 7 Jan 2025 17:25:48 -0800 Subject: [PATCH 405/427] GH-127809: Fix the JIT's understanding of ** (GH-127844) --- Lib/test/test_capi/test_opt.py | 44 +++++++++++ ...-12-11-14-32-22.gh-issue-127809.0W8khe.rst | 2 + Python/bytecodes.c | 16 ++++ Python/executor_cases.c.h | 16 ++++ Python/generated_cases.c.h | 16 ++++ Python/optimizer_bytecodes.c | 57 ++++++++++++--- Python/optimizer_cases.c.h | 73 +++++++++++++++---- Tools/cases_generator/analyzer.py | 1 + 8 files changed, 199 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-11-14-32-22.gh-issue-127809.0W8khe.rst diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 4cf9b66170c055..d84702411afe41 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1,4 +1,5 @@ import contextlib +import itertools import sys import textwrap import unittest @@ -1511,6 +1512,49 @@ def test_jit_error_pops(self): with self.assertRaises(TypeError): {item for item in items} + def test_power_type_depends_on_input_values(self): + template = textwrap.dedent(""" + import _testinternalcapi + + L, R, X, Y = {l}, {r}, {x}, {y} + + def check(actual: complex, expected: complex) -> None: + assert actual == expected, (actual, expected) + assert type(actual) is type(expected), (actual, expected) + + def f(l: complex, r: complex) -> None: + expected_local_local = pow(l, r) + pow(l, r) + expected_const_local = pow(L, r) + pow(L, r) + expected_local_const = pow(l, R) + pow(l, R) + expected_const_const = pow(L, R) + pow(L, R) + for _ in range(_testinternalcapi.TIER2_THRESHOLD): + # Narrow types: + l + l, r + r + # The powers produce results, and the addition is unguarded: + check(l ** r + l ** r, expected_local_local) + check(L ** r + L ** r, expected_const_local) + check(l ** R + l ** R, expected_local_const) + check(L ** R + L ** R, expected_const_const) + + # JIT for one pair of values... + f(L, R) + # ...then run with another: + f(X, Y) + """) + interesting = [ + (1, 1), # int ** int -> int + (1, -1), # int ** int -> float + (1.0, 1), # float ** int -> float + (1, 1.0), # int ** float -> float + (-1, 0.5), # int ** float -> complex + (1.0, 1.0), # float ** float -> float + (-1.0, 0.5), # float ** float -> complex + ] + for (l, r), (x, y) in itertools.product(interesting, repeat=2): + s = template.format(l=l, r=r, x=x, y=y) + with self.subTest(l=l, r=r, x=x, y=y): + script_helper.assert_python_ok("-c", s) + def global_identity(x): return x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-11-14-32-22.gh-issue-127809.0W8khe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-11-14-32-22.gh-issue-127809.0W8khe.rst new file mode 100644 index 00000000000000..19c8cc6e99c8c5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-11-14-32-22.gh-issue-127809.0W8khe.rst @@ -0,0 +1,2 @@ +Fix an issue where the experimental JIT may infer an incorrect result type +for exponentiation (``**`` and ``**=``), leading to bugs or crashes. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ec1cd00962ac0a..8bab4ea16b629b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -530,6 +530,8 @@ dummy_func( pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); @@ -543,6 +545,8 @@ dummy_func( pure op(_BINARY_OP_ADD_INT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); @@ -556,6 +560,8 @@ dummy_func( pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); @@ -593,6 +599,8 @@ dummy_func( pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = @@ -607,6 +615,8 @@ dummy_func( pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = @@ -621,6 +631,8 @@ dummy_func( pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = @@ -650,6 +662,8 @@ dummy_func( pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); @@ -672,6 +686,8 @@ dummy_func( op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); int next_oparg; #if TIER_ONE diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ac2f69b7e98dc3..e40fa88be89172 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -638,6 +638,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -658,6 +660,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -678,6 +682,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -738,6 +744,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval * @@ -759,6 +767,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval + @@ -780,6 +790,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval - @@ -819,6 +831,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -838,6 +852,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); int next_oparg; #if TIER_ONE assert(next_instr->op.code == STORE_FAST); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index eaa8a563464068..7028ba52faae96 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -80,6 +80,8 @@ { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval + @@ -116,6 +118,8 @@ { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -151,6 +155,8 @@ { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -185,6 +191,8 @@ { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); int next_oparg; #if TIER_ONE assert(next_instr->op.code == STORE_FAST); @@ -247,6 +255,8 @@ { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval * @@ -283,6 +293,8 @@ { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -318,6 +330,8 @@ { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval - @@ -354,6 +368,8 @@ { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index a14d119b7a1dec..86394480f76bb8 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -167,23 +167,56 @@ dummy_func(void) { } op(_BINARY_OP, (left, right -- res)) { - PyTypeObject *ltype = sym_get_type(left); - PyTypeObject *rtype = sym_get_type(right); - if (ltype != NULL && (ltype == &PyLong_Type || ltype == &PyFloat_Type) && - rtype != NULL && (rtype == &PyLong_Type || rtype == &PyFloat_Type)) - { - if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && - ltype == &PyLong_Type && rtype == &PyLong_Type) { - /* If both inputs are ints and the op is not division the result is an int */ - res = sym_new_type(ctx, &PyLong_Type); + bool lhs_int = sym_matches_type(left, &PyLong_Type); + bool rhs_int = sym_matches_type(right, &PyLong_Type); + bool lhs_float = sym_matches_type(left, &PyFloat_Type); + bool rhs_float = sym_matches_type(right, &PyFloat_Type); + if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { + // There's something other than an int or float involved: + res = sym_new_unknown(ctx); + } + else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { + // This one's fun... the *type* of the result depends on the + // *values* being exponentiated. However, exponents with one + // constant part are reasonably common, so it's probably worth + // trying to infer some simple cases: + // - A: 1 ** 1 -> 1 (int ** int -> int) + // - B: 1 ** -1 -> 1.0 (int ** int -> float) + // - C: 1.0 ** 1 -> 1.0 (float ** int -> float) + // - D: 1 ** 1.0 -> 1.0 (int ** float -> float) + // - E: -1 ** 0.5 ~> 1j (int ** float -> complex) + // - F: 1.0 ** 1.0 -> 1.0 (float ** float -> float) + // - G: -1.0 ** 0.5 ~> 1j (float ** float -> complex) + if (rhs_float) { + // Case D, E, F, or G... can't know without the sign of the LHS + // or whether the RHS is whole, which isn't worth the effort: + res = sym_new_unknown(ctx); } - else { - /* For any other op combining ints/floats the result is a float */ + else if (lhs_float) { + // Case C: res = sym_new_type(ctx, &PyFloat_Type); } + else if (!sym_is_const(right)) { + // Case A or B... can't know without the sign of the RHS: + res = sym_new_unknown(ctx); + } + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(right))) { + // Case B: + res = sym_new_type(ctx, &PyFloat_Type); + } + else { + // Case A: + res = sym_new_type(ctx, &PyLong_Type); + } + } + else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { + res = sym_new_type(ctx, &PyFloat_Type); + } + else if (lhs_int && rhs_int) { + res = sym_new_type(ctx, &PyLong_Type); } else { - res = sym_new_unknown(ctx); + res = sym_new_type(ctx, &PyFloat_Type); } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index be3e06108aec92..c72ae7b6281e80 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2307,24 +2307,69 @@ _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - PyTypeObject *ltype = sym_get_type(left); - PyTypeObject *rtype = sym_get_type(right); - if (ltype != NULL && (ltype == &PyLong_Type || ltype == &PyFloat_Type) && - rtype != NULL && (rtype == &PyLong_Type || rtype == &PyFloat_Type)) - { - if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && - ltype == &PyLong_Type && rtype == &PyLong_Type) { - /* If both inputs are ints and the op is not division the result is an int */ - res = sym_new_type(ctx, &PyLong_Type); + bool lhs_int = sym_matches_type(left, &PyLong_Type); + bool rhs_int = sym_matches_type(right, &PyLong_Type); + bool lhs_float = sym_matches_type(left, &PyFloat_Type); + bool rhs_float = sym_matches_type(right, &PyFloat_Type); + if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { + // There's something other than an int or float involved: + res = sym_new_unknown(ctx); + } + else { + if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { + // This one's fun... the *type* of the result depends on the + // *values* being exponentiated. However, exponents with one + // constant part are reasonably common, so it's probably worth + // trying to infer some simple cases: + // - A: 1 ** 1 -> 1 (int ** int -> int) + // - B: 1 ** -1 -> 1.0 (int ** int -> float) + // - C: 1.0 ** 1 -> 1.0 (float ** int -> float) + // - D: 1 ** 1.0 -> 1.0 (int ** float -> float) + // - E: -1 ** 0.5 ~> 1j (int ** float -> complex) + // - F: 1.0 ** 1.0 -> 1.0 (float ** float -> float) + // - G: -1.0 ** 0.5 ~> 1j (float ** float -> complex) + if (rhs_float) { + // Case D, E, F, or G... can't know without the sign of the LHS + // or whether the RHS is whole, which isn't worth the effort: + res = sym_new_unknown(ctx); + } + else { + if (lhs_float) { + // Case C: + res = sym_new_type(ctx, &PyFloat_Type); + } + else { + if (!sym_is_const(right)) { + // Case A or B... can't know without the sign of the RHS: + res = sym_new_unknown(ctx); + } + else { + if (_PyLong_IsNegative((PyLongObject *)sym_get_const(right))) { + // Case B: + res = sym_new_type(ctx, &PyFloat_Type); + } + else { + // Case A: + res = sym_new_type(ctx, &PyLong_Type); + } + } + } + } } else { - /* For any other op combining ints/floats the result is a float */ - res = sym_new_type(ctx, &PyFloat_Type); + if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { + res = sym_new_type(ctx, &PyFloat_Type); + } + else { + if (lhs_int && rhs_int) { + res = sym_new_type(ctx, &PyLong_Type); + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + } } } - else { - res = sym_new_unknown(ctx); - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c0a370a936aa94..679beca3ec3a9d 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -599,6 +599,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "_PyLong_CompactValue", "_PyLong_DigitCount", "_PyLong_IsCompact", + "_PyLong_IsNegative", "_PyLong_IsNonNegativeCompact", "_PyLong_IsZero", "_PyLong_Multiply", From 15372d0112ac7fbbbe313725dd87d5a45af2f6d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Srinivas=20Reddy=20Thatiparthy=20=28=E0=B0=A4=E0=B0=BE?= =?UTF-8?q?=E0=B0=9F=E0=B0=BF=E0=B0=AA=E0=B0=B0=E0=B1=8D=E0=B0=A4=E0=B0=BF?= =?UTF-8?q?=20=E0=B0=B6=E0=B1=8D=E0=B0=B0=E0=B1=80=E0=B0=A8=E0=B0=BF?= =?UTF-8?q?=E0=B0=B5=E0=B0=BE=E0=B0=B8=E0=B1=8D=20=20=E0=B0=B0=E0=B1=86?= =?UTF-8?q?=E0=B0=A1=E0=B1=8D=E0=B0=A1=E0=B0=BF=29?= Date: Wed, 8 Jan 2025 13:05:43 +0530 Subject: [PATCH 406/427] gh-87506: Document that json.load*() can raise UnicodeDecodeError (#127355) Co-authored-by: Erlend Aasland --- Doc/library/json.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index cf516cf3dc1d29..41a09b40d7e88b 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -324,6 +324,10 @@ Basic Usage :raises JSONDecodeError: When the data being deserialized is not a valid JSON document. + :raises UnicodeDecodeError: + When the data being deserialized does not contain + UTF-8, UTF-16 or UTF-32 encoded data. + .. versionchanged:: 3.1 * Added the optional *object_pairs_hook* parameter. From cdfb8bc93a4d8c06d2404ba2d243937ba209438c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 8 Jan 2025 09:36:44 +0100 Subject: [PATCH 407/427] gh-87506: Amend json.loads() post PR #127355 (#128609) --- Doc/library/json.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 41a09b40d7e88b..4e7046d6d8f6ac 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -347,15 +347,11 @@ Basic Usage .. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) - Deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` + Identical to :func:`load`, but instead of a file-like object, + deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` instance containing a JSON document) to a Python object using this :ref:`conversion table `. - The other arguments have the same meaning as in :func:`load`. - - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. - .. versionchanged:: 3.6 *s* can now be of type :class:`bytes` or :class:`bytearray`. The input encoding should be UTF-8, UTF-16 or UTF-32. From 474e419792484d1c16e7d9c99b7bf144136b9307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Srinivas=20Reddy=20Thatiparthy=20=28=E0=B0=A4=E0=B0=BE?= =?UTF-8?q?=E0=B0=9F=E0=B0=BF=E0=B0=AA=E0=B0=B0=E0=B1=8D=E0=B0=A4=E0=B0=BF?= =?UTF-8?q?=20=E0=B0=B6=E0=B1=8D=E0=B0=B0=E0=B1=80=E0=B0=A8=E0=B0=BF?= =?UTF-8?q?=E0=B0=B5=E0=B0=BE=E0=B0=B8=E0=B1=8D=20=20=E0=B0=B0=E0=B1=86?= =?UTF-8?q?=E0=B0=A1=E0=B1=8D=E0=B0=A1=E0=B0=BF=29?= Date: Wed, 8 Jan 2025 16:02:07 +0530 Subject: [PATCH 408/427] gh-41872: Fix quick extraction of module docstrings from a file in pydoc (GH-127520) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It now supports docstrings with single quotes, escape sequences, raw string literals, and other Python syntax. Co-authored-by: Éric Co-authored-by: Serhiy Storchaka --- Lib/pydoc.py | 39 ++++++---- Lib/test/test_pydoc/test_pydoc.py | 77 +++++++++++++++++++ ...4-12-17-15-23-40.gh-issue-41872.31LjKY.rst | 3 + 3 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index c863794ea14ef9..9e84292aaf825f 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -53,6 +53,7 @@ class or function within a module or module in a package. If the # the current directory is changed with os.chdir(), an incorrect # path will be displayed. +import ast import __future__ import builtins import importlib._bootstrap @@ -384,21 +385,29 @@ def ispackage(path): return False def source_synopsis(file): - line = file.readline() - while line[:1] == '#' or not line.strip(): - line = file.readline() - if not line: break - line = line.strip() - if line[:4] == 'r"""': line = line[1:] - if line[:3] == '"""': - line = line[3:] - if line[-1:] == '\\': line = line[:-1] - while not line.strip(): - line = file.readline() - if not line: break - result = line.split('"""')[0].strip() - else: result = None - return result + """Return the one-line summary of a file object, if present""" + + string = '' + try: + tokens = tokenize.generate_tokens(file.readline) + for tok_type, tok_string, _, _, _ in tokens: + if tok_type == tokenize.STRING: + string += tok_string + elif tok_type == tokenize.NEWLINE: + with warnings.catch_warnings(): + # Ignore the "invalid escape sequence" warning. + warnings.simplefilter("ignore", SyntaxWarning) + docstring = ast.literal_eval(string) + if not isinstance(docstring, str): + return None + return docstring.strip().split('\n')[0].strip() + elif tok_type == tokenize.OP and tok_string in ('(', ')'): + string += tok_string + elif tok_type not in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING): + return None + except (tokenize.TokenError, UnicodeDecodeError, SyntaxError): + return None + return None def synopsis(filename, cache={}): """Get the one-line summary out of a module file.""" diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index c798b11f5aa56e..cec18aa9440c9e 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -4,6 +4,7 @@ import contextlib import importlib.util import inspect +import io import pydoc import py_compile import keyword @@ -899,6 +900,82 @@ def test_synopsis(self): synopsis = pydoc.synopsis(TESTFN, {}) self.assertEqual(synopsis, 'line 1: h\xe9') + def test_source_synopsis(self): + def check(source, expected, encoding=None): + if isinstance(source, str): + source_file = StringIO(source) + else: + source_file = io.TextIOWrapper(io.BytesIO(source), encoding=encoding) + with source_file: + result = pydoc.source_synopsis(source_file) + self.assertEqual(result, expected) + + check('"""Single line docstring."""', + 'Single line docstring.') + check('"""First line of docstring.\nSecond line.\nThird line."""', + 'First line of docstring.') + check('"""First line of docstring.\\nSecond line.\\nThird line."""', + 'First line of docstring.') + check('""" Whitespace around docstring. """', + 'Whitespace around docstring.') + check('import sys\n"""No docstring"""', + None) + check(' \n"""Docstring after empty line."""', + 'Docstring after empty line.') + check('# Comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check(' # Indented comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check('""""""', # Empty docstring + '') + check('', # Empty file + None) + check('"""Embedded\0null byte"""', + None) + check('"""Embedded null byte"""\0', + None) + check('"""Café and résumé."""', + 'Café and résumé.') + check("'''Triple single quotes'''", + 'Triple single quotes') + check('"Single double quotes"', + 'Single double quotes') + check("'Single single quotes'", + 'Single single quotes') + check('"""split\\\nline"""', + 'splitline') + check('"""Unrecognized escape \\sequence"""', + 'Unrecognized escape \\sequence') + check('"""Invalid escape seq\\uence"""', + None) + check('r"""Raw \\stri\\ng"""', + 'Raw \\stri\\ng') + check('b"""Bytes literal"""', + None) + check('f"""f-string"""', + None) + check('"""Concatenated""" \\\n"string" \'literals\'', + 'Concatenatedstringliterals') + check('"""String""" + """expression"""', + None) + check('("""In parentheses""")', + 'In parentheses') + check('("""Multiple lines """\n"""in parentheses""")', + 'Multiple lines in parentheses') + check('()', # tuple + None) + check(b'# coding: iso-8859-15\n"""\xa4uro sign"""', + '€uro sign', encoding='iso-8859-15') + check(b'"""\xa4"""', # Decoding error + None, encoding='utf-8') + + with tempfile.NamedTemporaryFile(mode='w+', encoding='utf-8') as temp_file: + temp_file.write('"""Real file test."""\n') + temp_file.flush() + temp_file.seek(0) + result = pydoc.source_synopsis(temp_file) + self.assertEqual(result, "Real file test.") + @requires_docstrings def test_synopsis_sourceless(self): os = import_helper.import_fresh_module('os') diff --git a/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst b/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst new file mode 100644 index 00000000000000..b807dcb284c248 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst @@ -0,0 +1,3 @@ +Fix quick extraction of module docstrings from a file in :mod:`pydoc`. +It now supports docstrings with single quotes, escape sequences, +raw string literals, and other Python syntax. From 971a52b5495e3d596e599faa1f31d4671897026d Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 8 Jan 2025 14:19:41 +0300 Subject: [PATCH 409/427] gh-128617: Fix `test_typing.test_readonly_inheritance` (#128618) --- Lib/test/test_typing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index ef3cfc9517085e..0bb9ada221e985 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -8912,13 +8912,13 @@ class Child1(Base1): self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) class Base2(TypedDict): - a: ReadOnly[int] + a: int class Child2(Base2): - b: str + b: ReadOnly[str] - self.assertEqual(Child1.__readonly_keys__, frozenset({'a'})) - self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) + self.assertEqual(Child2.__readonly_keys__, frozenset({'b'})) + self.assertEqual(Child2.__mutable_keys__, frozenset({'a'})) def test_cannot_make_mutable_key_readonly(self): class Base(TypedDict): From eb26e170695f15714b5e2ae0c0b83aa790c97869 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 8 Jan 2025 15:02:47 +0300 Subject: [PATCH 410/427] gh-128613: Increase `typing.Concatenate` coverage (#128614) --- Lib/test/test_typing.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 0bb9ada221e985..a75dac4a6102bf 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -10129,6 +10129,18 @@ def test_valid_uses(self): self.assertEqual(C4.__args__, (Concatenate[int, T, P], T)) self.assertEqual(C4.__parameters__, (T, P)) + def test_invalid_uses(self): + with self.assertRaisesRegex(TypeError, 'Concatenate of no types'): + Concatenate[()] + with self.assertRaisesRegex( + TypeError, + ( + 'The last parameter to Concatenate should be a ' + 'ParamSpec variable or ellipsis' + ), + ): + Concatenate[int] + def test_var_substitution(self): T = TypeVar('T') P = ParamSpec('P') From 95cd9c669cdc7718198addb1abb49941a2c61fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Wed, 8 Jan 2025 12:03:21 +0000 Subject: [PATCH 411/427] GH-127970: find the runtime library when dladdr is available (#127972) --- ...-12-15-19-51-54.gh-issue-127970.vdUp-y.rst | 6 +++ Modules/getpath.c | 38 ++++++++----------- configure | 6 +++ configure.ac | 2 +- pyconfig.h.in | 3 ++ 5 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-19-51-54.gh-issue-127970.vdUp-y.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-19-51-54.gh-issue-127970.vdUp-y.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-19-51-54.gh-issue-127970.vdUp-y.rst new file mode 100644 index 00000000000000..e4dc7b5fe032d6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-19-51-54.gh-issue-127970.vdUp-y.rst @@ -0,0 +1,6 @@ +We now use the location of the ``libpython`` runtime library used in the current +proccess to determine :data:`sys.base_prefix` on all platforms implementing the +`dladdr `_ +function defined by the UNIX standard — this includes Linux, Android, macOS, +iOS, FreeBSD, etc. This was already the case on Windows and macOS Framework +builds. diff --git a/Modules/getpath.c b/Modules/getpath.c index 18ddfaf8dbce1a..2d3c9757298d16 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -17,10 +17,13 @@ #endif #ifdef __APPLE__ -# include # include #endif +#ifdef HAVE_DLFCN_H +# include +#endif + /* Reference the precompiled getpath.py */ #include "Python/frozen_modules/getpath.h" @@ -803,36 +806,25 @@ progname_to_dict(PyObject *dict, const char *key) static int library_to_dict(PyObject *dict, const char *key) { +/* macOS framework builds do not link against a libpython dynamic library, but + instead link against a macOS Framework. */ +#if defined(Py_ENABLE_SHARED) || defined(WITH_NEXT_FRAMEWORK) + #ifdef MS_WINDOWS -#ifdef Py_ENABLE_SHARED extern HMODULE PyWin_DLLhModule; if (PyWin_DLLhModule) { return winmodule_to_dict(dict, key, PyWin_DLLhModule); } #endif -#elif defined(WITH_NEXT_FRAMEWORK) - static char modPath[MAXPATHLEN + 1]; - static int modPathInitialized = -1; - if (modPathInitialized < 0) { - modPathInitialized = 0; - - /* On Mac OS X we have a special case if we're running from a framework. - This is because the python home should be set relative to the library, - which is in the framework, not relative to the executable, which may - be outside of the framework. Except when we're in the build - directory... */ - Dl_info pythonInfo; - if (dladdr(&Py_Initialize, &pythonInfo)) { - if (pythonInfo.dli_fname) { - strncpy(modPath, pythonInfo.dli_fname, MAXPATHLEN); - modPathInitialized = 1; - } - } - } - if (modPathInitialized > 0) { - return decode_to_dict(dict, key, modPath); + +#if HAVE_DLADDR + Dl_info libpython_info; + if (dladdr(&Py_Initialize, &libpython_info) && libpython_info.dli_fname) { + return decode_to_dict(dict, key, libpython_info.dli_fname); } #endif +#endif + return PyDict_SetItemString(dict, key, Py_None) == 0; } diff --git a/configure b/configure index 6e1b393a3ece68..bb77c558abda5a 100755 --- a/configure +++ b/configure @@ -19001,6 +19001,12 @@ if test "x$ac_cv_func_ctermid" = xyes then : printf "%s\n" "#define HAVE_CTERMID 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "dladdr" "ac_cv_func_dladdr" +if test "x$ac_cv_func_dladdr" = xyes +then : + printf "%s\n" "#define HAVE_DLADDR 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "dup" "ac_cv_func_dup" if test "x$ac_cv_func_dup" = xyes diff --git a/configure.ac b/configure.ac index 6d44be8959865d..653cd3f6c531b6 100644 --- a/configure.ac +++ b/configure.ac @@ -5128,7 +5128,7 @@ fi # checks for library functions AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ - copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ + copy_file_range ctermid dladdr dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ diff --git a/pyconfig.h.in b/pyconfig.h.in index 874b98dc96585a..aaf52168c3d39d 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -286,6 +286,9 @@ /* Define if you have the 'dirfd' function or macro. */ #undef HAVE_DIRFD +/* Define to 1 if you have the 'dladdr' function. */ +#undef HAVE_DLADDR + /* Define to 1 if you have the header file. */ #undef HAVE_DLFCN_H From 74a517181a9bb65a1f6da149af7427a9fcb3add3 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 8 Jan 2025 15:04:54 +0300 Subject: [PATCH 412/427] gh-128615: Cover pickling of `ParamSpecArgs` and `ParamSpecKwargs` (#128616) --- Lib/test/test_typing.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a75dac4a6102bf..45ba7611059e43 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -5182,6 +5182,18 @@ class C(B[int]): x = pickle.loads(z) self.assertEqual(s, x) + # Test ParamSpec args and kwargs + global PP + PP = ParamSpec('PP') + for thing in [PP.args, PP.kwargs]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + self.assertEqual( + pickle.loads(pickle.dumps(thing, proto)), + thing, + ) + del PP + def test_copy_and_deepcopy(self): T = TypeVar('T') class Node(Generic[T]): ... From c22302ecea8c22379a41cd77158748d5fcd91352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:50:40 +0100 Subject: [PATCH 413/427] gh-111178: fix UBSan failures in `Python/hamt.c` (GH-128247) * fix UBSan failures for `PyHamtObject` * fix UBSan failures for `PyHamtNode_Array` * fix UBSan failures for `PyHamtNode_Collision` * fix UBSan failures for `PyHamtNode_Bitmap` --- Python/hamt.c | 174 ++++++++++++++++++++++++-------------------------- 1 file changed, 84 insertions(+), 90 deletions(-) diff --git a/Python/hamt.c b/Python/hamt.c index cfd211f4541446..ed43a0449d7a01 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -319,6 +319,8 @@ typedef struct { Py_ssize_t a_count; } PyHamtNode_Array; +#define _PyHamtNode_Array_CAST(op) ((PyHamtNode_Array *)(op)) + typedef struct { PyObject_VAR_HEAD @@ -326,6 +328,8 @@ typedef struct { PyObject *c_array[1]; } PyHamtNode_Collision; +#define _PyHamtNode_Collision_CAST(op) ((PyHamtNode_Collision *)(op)) + static PyHamtObject * hamt_alloc(void); @@ -479,6 +483,8 @@ _hamt_dump_ident(PyUnicodeWriter *writer, int level) #endif /* Py_DEBUG */ /////////////////////////////////// Bitmap Node +#define _PyHamtNode_Bitmap_CAST(op) ((PyHamtNode_Bitmap *)(op)) + static PyHamtNode * hamt_node_bitmap_new(Py_ssize_t size) @@ -1083,30 +1089,27 @@ hamt_node_bitmap_find(PyHamtNode_Bitmap *self, } static int -hamt_node_bitmap_traverse(PyHamtNode_Bitmap *self, visitproc visit, void *arg) +hamt_node_bitmap_traverse(PyObject *op, visitproc visit, void *arg) { /* Bitmap's tp_traverse */ - - Py_ssize_t i; - - for (i = Py_SIZE(self); --i >= 0; ) { + PyHamtNode_Bitmap *self = _PyHamtNode_Bitmap_CAST(op); + for (Py_ssize_t i = Py_SIZE(self); --i >= 0;) { Py_VISIT(self->b_array[i]); } - return 0; } static void -hamt_node_bitmap_dealloc(PyHamtNode_Bitmap *self) +hamt_node_bitmap_dealloc(PyObject *self) { /* Bitmap's tp_dealloc */ - Py_ssize_t len = Py_SIZE(self); - Py_ssize_t i; + PyHamtNode_Bitmap *node = _PyHamtNode_Bitmap_CAST(self); + Py_ssize_t i, len = Py_SIZE(self); - if (Py_SIZE(self) == 0) { + if (len == 0) { /* The empty node is statically allocated. */ - assert(self == &_Py_SINGLETON(hamt_bitmap_node_empty)); + assert(node == &_Py_SINGLETON(hamt_bitmap_node_empty)); #ifdef Py_DEBUG _Py_FatalRefcountError("deallocating the empty hamt node bitmap singleton"); #else @@ -1120,11 +1123,11 @@ hamt_node_bitmap_dealloc(PyHamtNode_Bitmap *self) if (len > 0) { i = len; while (--i >= 0) { - Py_XDECREF(self->b_array[i]); + Py_XDECREF(node->b_array[i]); } } - Py_TYPE(self)->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free(self); Py_TRASHCAN_END } @@ -1489,38 +1492,30 @@ hamt_node_collision_find(PyHamtNode_Collision *self, static int -hamt_node_collision_traverse(PyHamtNode_Collision *self, - visitproc visit, void *arg) +hamt_node_collision_traverse(PyObject *op, visitproc visit, void *arg) { /* Collision's tp_traverse */ - - Py_ssize_t i; - - for (i = Py_SIZE(self); --i >= 0; ) { + PyHamtNode_Collision *self = _PyHamtNode_Collision_CAST(op); + for (Py_ssize_t i = Py_SIZE(self); --i >= 0; ) { Py_VISIT(self->c_array[i]); } - return 0; } static void -hamt_node_collision_dealloc(PyHamtNode_Collision *self) +hamt_node_collision_dealloc(PyObject *self) { /* Collision's tp_dealloc */ - Py_ssize_t len = Py_SIZE(self); - PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, hamt_node_collision_dealloc) - if (len > 0) { - + PyHamtNode_Collision *node = _PyHamtNode_Collision_CAST(self); while (--len >= 0) { - Py_XDECREF(self->c_array[len]); + Py_XDECREF(node->c_array[len]); } } - - Py_TYPE(self)->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free(self); Py_TRASHCAN_END } @@ -1868,35 +1863,27 @@ hamt_node_array_find(PyHamtNode_Array *self, } static int -hamt_node_array_traverse(PyHamtNode_Array *self, - visitproc visit, void *arg) +hamt_node_array_traverse(PyObject *op, visitproc visit, void *arg) { /* Array's tp_traverse */ - - Py_ssize_t i; - - for (i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { + PyHamtNode_Array *self = _PyHamtNode_Array_CAST(op); + for (Py_ssize_t i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { Py_VISIT(self->a_array[i]); } - return 0; } static void -hamt_node_array_dealloc(PyHamtNode_Array *self) +hamt_node_array_dealloc(PyObject *self) { /* Array's tp_dealloc */ - - Py_ssize_t i; - PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, hamt_node_array_dealloc) - - for (i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { - Py_XDECREF(self->a_array[i]); + PyHamtNode_Array *obj = _PyHamtNode_Array_CAST(self); + for (Py_ssize_t i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { + Py_XDECREF(obj->a_array[i]); } - - Py_TYPE(self)->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free(self); Py_TRASHCAN_END } @@ -2605,6 +2592,8 @@ static PyObject * hamt_dump(PyHamtObject *self); #endif +#define _PyHamtObject_CAST(op) ((PyHamtObject *)(op)) + static PyObject * hamt_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -2613,24 +2602,27 @@ hamt_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -hamt_tp_clear(PyHamtObject *self) +hamt_tp_clear(PyObject *op) { + PyHamtObject *self = _PyHamtObject_CAST(op); Py_CLEAR(self->h_root); return 0; } static int -hamt_tp_traverse(PyHamtObject *self, visitproc visit, void *arg) +hamt_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyHamtObject *self = _PyHamtObject_CAST(op); Py_VISIT(self->h_root); return 0; } static void -hamt_tp_dealloc(PyHamtObject *self) +hamt_tp_dealloc(PyObject *self) { - if (self == _empty_hamt) { + PyHamtObject *obj = _PyHamtObject_CAST(self); + if (obj == _empty_hamt) { /* The empty one is statically allocated. */ #ifdef Py_DEBUG _Py_FatalRefcountError("deallocating the empty hamt singleton"); @@ -2640,8 +2632,8 @@ hamt_tp_dealloc(PyHamtObject *self) } PyObject_GC_UnTrack(self); - if (self->h_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)self); + if (obj->h_weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } (void)hamt_tp_clear(self); Py_TYPE(self)->tp_free(self); @@ -2673,16 +2665,18 @@ hamt_tp_richcompare(PyObject *v, PyObject *w, int op) } static int -hamt_tp_contains(PyHamtObject *self, PyObject *key) +hamt_tp_contains(PyObject *op, PyObject *key) { PyObject *val; + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_Find(self, key, &val); } static PyObject * -hamt_tp_subscript(PyHamtObject *self, PyObject *key) +hamt_tp_subscript(PyObject *op, PyObject *key) { PyObject *val; + PyHamtObject *self = _PyHamtObject_CAST(op); hamt_find_t res = hamt_find(self, key, &val); switch (res) { case F_ERROR: @@ -2698,19 +2692,21 @@ hamt_tp_subscript(PyHamtObject *self, PyObject *key) } static Py_ssize_t -hamt_tp_len(PyHamtObject *self) +hamt_tp_len(PyObject *op) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_Len(self); } static PyObject * -hamt_tp_iter(PyHamtObject *self) +hamt_tp_iter(PyObject *op) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterKeys(self); } static PyObject * -hamt_py_set(PyHamtObject *self, PyObject *args) +hamt_py_set(PyObject *op, PyObject *args) { PyObject *key; PyObject *val; @@ -2719,11 +2715,12 @@ hamt_py_set(PyHamtObject *self, PyObject *args) return NULL; } + PyHamtObject *self = _PyHamtObject_CAST(op); return (PyObject *)_PyHamt_Assoc(self, key, val); } static PyObject * -hamt_py_get(PyHamtObject *self, PyObject *args) +hamt_py_get(PyObject *op, PyObject *args) { PyObject *key; PyObject *def = NULL; @@ -2733,6 +2730,7 @@ hamt_py_get(PyHamtObject *self, PyObject *args) } PyObject *val = NULL; + PyHamtObject *self = _PyHamtObject_CAST(op); hamt_find_t res = hamt_find(self, key, &val); switch (res) { case F_ERROR: @@ -2750,67 +2748,63 @@ hamt_py_get(PyHamtObject *self, PyObject *args) } static PyObject * -hamt_py_delete(PyHamtObject *self, PyObject *key) +hamt_py_delete(PyObject *op, PyObject *key) { + PyHamtObject *self = _PyHamtObject_CAST(op); return (PyObject *)_PyHamt_Without(self, key); } static PyObject * -hamt_py_items(PyHamtObject *self, PyObject *args) +hamt_py_items(PyObject *op, PyObject *args) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterItems(self); } static PyObject * -hamt_py_values(PyHamtObject *self, PyObject *args) +hamt_py_values(PyObject *op, PyObject *args) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterValues(self); } static PyObject * -hamt_py_keys(PyHamtObject *self, PyObject *Py_UNUSED(args)) +hamt_py_keys(PyObject *op, PyObject *Py_UNUSED(args)) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterKeys(self); } #ifdef Py_DEBUG static PyObject * -hamt_py_dump(PyHamtObject *self, PyObject *Py_UNUSED(args)) +hamt_py_dump(PyObject *op, PyObject *Py_UNUSED(args)) { + PyHamtObject *self = _PyHamtObject_CAST(op); return hamt_dump(self); } #endif static PyMethodDef PyHamt_methods[] = { - {"set", _PyCFunction_CAST(hamt_py_set), METH_VARARGS, NULL}, - {"get", _PyCFunction_CAST(hamt_py_get), METH_VARARGS, NULL}, - {"delete", _PyCFunction_CAST(hamt_py_delete), METH_O, NULL}, - {"items", _PyCFunction_CAST(hamt_py_items), METH_NOARGS, NULL}, - {"keys", _PyCFunction_CAST(hamt_py_keys), METH_NOARGS, NULL}, - {"values", _PyCFunction_CAST(hamt_py_values), METH_NOARGS, NULL}, + {"set", hamt_py_set, METH_VARARGS, NULL}, + {"get", hamt_py_get, METH_VARARGS, NULL}, + {"delete", hamt_py_delete, METH_O, NULL}, + {"items", hamt_py_items, METH_NOARGS, NULL}, + {"keys", hamt_py_keys, METH_NOARGS, NULL}, + {"values", hamt_py_values, METH_NOARGS, NULL}, #ifdef Py_DEBUG - {"__dump__", _PyCFunction_CAST(hamt_py_dump), METH_NOARGS, NULL}, + {"__dump__", hamt_py_dump, METH_NOARGS, NULL}, #endif {NULL, NULL} }; static PySequenceMethods PyHamt_as_sequence = { - 0, /* sq_length */ - 0, /* sq_concat */ - 0, /* sq_repeat */ - 0, /* sq_item */ - 0, /* sq_slice */ - 0, /* sq_ass_item */ - 0, /* sq_ass_slice */ - (objobjproc)hamt_tp_contains, /* sq_contains */ - 0, /* sq_inplace_concat */ - 0, /* sq_inplace_repeat */ + .sq_contains = hamt_tp_contains, }; static PyMappingMethods PyHamt_as_mapping = { - (lenfunc)hamt_tp_len, /* mp_length */ - (binaryfunc)hamt_tp_subscript, /* mp_subscript */ + .mp_length = hamt_tp_len, + .mp_subscript = hamt_tp_subscript, }; PyTypeObject _PyHamt_Type = { @@ -2820,13 +2814,13 @@ PyTypeObject _PyHamt_Type = { .tp_methods = PyHamt_methods, .tp_as_mapping = &PyHamt_as_mapping, .tp_as_sequence = &PyHamt_as_sequence, - .tp_iter = (getiterfunc)hamt_tp_iter, - .tp_dealloc = (destructor)hamt_tp_dealloc, + .tp_iter = hamt_tp_iter, + .tp_dealloc = hamt_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_richcompare = hamt_tp_richcompare, - .tp_traverse = (traverseproc)hamt_tp_traverse, - .tp_clear = (inquiry)hamt_tp_clear, + .tp_traverse = hamt_tp_traverse, + .tp_clear = hamt_tp_clear, .tp_new = hamt_tp_new, .tp_weaklistoffset = offsetof(PyHamtObject, h_weakreflist), .tp_hash = PyObject_HashNotImplemented, @@ -2841,10 +2835,10 @@ PyTypeObject _PyHamt_ArrayNode_Type = { "hamt_array_node", sizeof(PyHamtNode_Array), 0, - .tp_dealloc = (destructor)hamt_node_array_dealloc, + .tp_dealloc = hamt_node_array_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)hamt_node_array_traverse, + .tp_traverse = hamt_node_array_traverse, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, }; @@ -2854,10 +2848,10 @@ PyTypeObject _PyHamt_BitmapNode_Type = { "hamt_bitmap_node", sizeof(PyHamtNode_Bitmap) - sizeof(PyObject *), sizeof(PyObject *), - .tp_dealloc = (destructor)hamt_node_bitmap_dealloc, + .tp_dealloc = hamt_node_bitmap_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)hamt_node_bitmap_traverse, + .tp_traverse = hamt_node_bitmap_traverse, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, }; @@ -2867,10 +2861,10 @@ PyTypeObject _PyHamt_CollisionNode_Type = { "hamt_collision_node", sizeof(PyHamtNode_Collision) - sizeof(PyObject *), sizeof(PyObject *), - .tp_dealloc = (destructor)hamt_node_collision_dealloc, + .tp_dealloc = hamt_node_collision_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)hamt_node_collision_traverse, + .tp_traverse = hamt_node_collision_traverse, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, }; From 1da0901894d7c0d56ebce97cd0c16aeffb64adcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:52:27 +0100 Subject: [PATCH 414/427] gh-111178: fix UBSan failures in `Python/context.c` (GH-128242) * fix UBSan failures for `PyContext` * fix UBSan failures for `PyContextVar` * fix UBSan failures for `PyContextToken` * fix UBSan failures for `_PyContextTokenMissing` --- Python/context.c | 121 ++++++++++++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 53 deletions(-) diff --git a/Python/context.c b/Python/context.c index 95aa82206270f9..f30b59b9443bbf 100644 --- a/Python/context.c +++ b/Python/context.c @@ -419,6 +419,9 @@ class _contextvars.Context "PyContext *" "&PyContext_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=bdf87f8e0cb580e8]*/ +#define _PyContext_CAST(op) ((PyContext *)(op)) + + static inline PyContext * _context_alloc(void) { @@ -513,28 +516,30 @@ context_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -context_tp_clear(PyContext *self) +context_tp_clear(PyObject *op) { + PyContext *self = _PyContext_CAST(op); Py_CLEAR(self->ctx_prev); Py_CLEAR(self->ctx_vars); return 0; } static int -context_tp_traverse(PyContext *self, visitproc visit, void *arg) +context_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyContext *self = _PyContext_CAST(op); Py_VISIT(self->ctx_prev); Py_VISIT(self->ctx_vars); return 0; } static void -context_tp_dealloc(PyContext *self) +context_tp_dealloc(PyObject *self) { _PyObject_GC_UNTRACK(self); - - if (self->ctx_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)self); + PyContext *ctx = _PyContext_CAST(self); + if (ctx->ctx_weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } (void)context_tp_clear(self); @@ -542,8 +547,9 @@ context_tp_dealloc(PyContext *self) } static PyObject * -context_tp_iter(PyContext *self) +context_tp_iter(PyObject *op) { + PyContext *self = _PyContext_CAST(op); return _PyHamt_NewIterKeys(self->ctx_vars); } @@ -575,18 +581,20 @@ context_tp_richcompare(PyObject *v, PyObject *w, int op) } static Py_ssize_t -context_tp_len(PyContext *self) +context_tp_len(PyObject *op) { + PyContext *self = _PyContext_CAST(op); return _PyHamt_Len(self->ctx_vars); } static PyObject * -context_tp_subscript(PyContext *self, PyObject *key) +context_tp_subscript(PyObject *op, PyObject *key) { if (context_check_key_type(key)) { return NULL; } PyObject *val = NULL; + PyContext *self = _PyContext_CAST(op); int found = _PyHamt_Find(self->ctx_vars, key, &val); if (found < 0) { return NULL; @@ -599,12 +607,13 @@ context_tp_subscript(PyContext *self, PyObject *key) } static int -context_tp_contains(PyContext *self, PyObject *key) +context_tp_contains(PyObject *op, PyObject *key) { if (context_check_key_type(key)) { return -1; } PyObject *val = NULL; + PyContext *self = _PyContext_CAST(op); return _PyHamt_Find(self->ctx_vars, key, &val); } @@ -701,7 +710,7 @@ _contextvars_Context_copy_impl(PyContext *self) static PyObject * -context_run(PyContext *self, PyObject *const *args, +context_run(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyThreadState *ts = _PyThreadState_GET(); @@ -712,14 +721,14 @@ context_run(PyContext *self, PyObject *const *args, return NULL; } - if (_PyContext_Enter(ts, (PyObject *)self)) { + if (_PyContext_Enter(ts, self)) { return NULL; } PyObject *call_result = _PyObject_VectorcallTstate( ts, args[0], args + 1, nargs - 1, kwnames); - if (_PyContext_Exit(ts, (PyObject *)self)) { + if (_PyContext_Exit(ts, self)) { Py_XDECREF(call_result); return NULL; } @@ -739,21 +748,12 @@ static PyMethodDef PyContext_methods[] = { }; static PySequenceMethods PyContext_as_sequence = { - 0, /* sq_length */ - 0, /* sq_concat */ - 0, /* sq_repeat */ - 0, /* sq_item */ - 0, /* sq_slice */ - 0, /* sq_ass_item */ - 0, /* sq_ass_slice */ - (objobjproc)context_tp_contains, /* sq_contains */ - 0, /* sq_inplace_concat */ - 0, /* sq_inplace_repeat */ + .sq_contains = context_tp_contains }; static PyMappingMethods PyContext_as_mapping = { - (lenfunc)context_tp_len, /* mp_length */ - (binaryfunc)context_tp_subscript, /* mp_subscript */ + .mp_length = context_tp_len, + .mp_subscript = context_tp_subscript }; PyTypeObject PyContext_Type = { @@ -763,13 +763,13 @@ PyTypeObject PyContext_Type = { .tp_methods = PyContext_methods, .tp_as_mapping = &PyContext_as_mapping, .tp_as_sequence = &PyContext_as_sequence, - .tp_iter = (getiterfunc)context_tp_iter, - .tp_dealloc = (destructor)context_tp_dealloc, + .tp_iter = context_tp_iter, + .tp_dealloc = context_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_richcompare = context_tp_richcompare, - .tp_traverse = (traverseproc)context_tp_traverse, - .tp_clear = (inquiry)context_tp_clear, + .tp_traverse = context_tp_traverse, + .tp_clear = context_tp_clear, .tp_new = context_tp_new, .tp_weaklistoffset = offsetof(PyContext, ctx_weakreflist), .tp_hash = PyObject_HashNotImplemented, @@ -909,6 +909,9 @@ class _contextvars.ContextVar "PyContextVar *" "&PyContextVar_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=445da935fa8883c3]*/ +#define _PyContextVar_CAST(op) ((PyContextVar *)(op)) + + static PyObject * contextvar_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -926,8 +929,9 @@ contextvar_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -contextvar_tp_clear(PyContextVar *self) +contextvar_tp_clear(PyObject *op) { + PyContextVar *self = _PyContextVar_CAST(op); Py_CLEAR(self->var_name); Py_CLEAR(self->var_default); #ifndef Py_GIL_DISABLED @@ -939,15 +943,16 @@ contextvar_tp_clear(PyContextVar *self) } static int -contextvar_tp_traverse(PyContextVar *self, visitproc visit, void *arg) +contextvar_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyContextVar *self = _PyContextVar_CAST(op); Py_VISIT(self->var_name); Py_VISIT(self->var_default); return 0; } static void -contextvar_tp_dealloc(PyContextVar *self) +contextvar_tp_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); (void)contextvar_tp_clear(self); @@ -955,14 +960,16 @@ contextvar_tp_dealloc(PyContextVar *self) } static Py_hash_t -contextvar_tp_hash(PyContextVar *self) +contextvar_tp_hash(PyObject *op) { + PyContextVar *self = _PyContextVar_CAST(op); return self->var_hash; } static PyObject * -contextvar_tp_repr(PyContextVar *self) +contextvar_tp_repr(PyObject *op) { + PyContextVar *self = _PyContextVar_CAST(op); // Estimation based on the shortest name and default value, // but maximize the pointer size. // "" @@ -1106,15 +1113,15 @@ PyTypeObject PyContextVar_Type = { sizeof(PyContextVar), .tp_methods = PyContextVar_methods, .tp_members = PyContextVar_members, - .tp_dealloc = (destructor)contextvar_tp_dealloc, + .tp_dealloc = contextvar_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)contextvar_tp_traverse, - .tp_clear = (inquiry)contextvar_tp_clear, + .tp_traverse = contextvar_tp_traverse, + .tp_clear = contextvar_tp_clear, .tp_new = contextvar_tp_new, .tp_free = PyObject_GC_Del, - .tp_hash = (hashfunc)contextvar_tp_hash, - .tp_repr = (reprfunc)contextvar_tp_repr, + .tp_hash = contextvar_tp_hash, + .tp_repr = contextvar_tp_repr, }; @@ -1129,6 +1136,9 @@ class _contextvars.Token "PyContextToken *" "&PyContextToken_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=338a5e2db13d3f5b]*/ +#define _PyContextToken_CAST(op) ((PyContextToken *)(op)) + + static PyObject * token_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1138,8 +1148,9 @@ token_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -token_tp_clear(PyContextToken *self) +token_tp_clear(PyObject *op) { + PyContextToken *self = _PyContextToken_CAST(op); Py_CLEAR(self->tok_ctx); Py_CLEAR(self->tok_var); Py_CLEAR(self->tok_oldval); @@ -1147,8 +1158,9 @@ token_tp_clear(PyContextToken *self) } static int -token_tp_traverse(PyContextToken *self, visitproc visit, void *arg) +token_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyContextToken *self = _PyContextToken_CAST(op); Py_VISIT(self->tok_ctx); Py_VISIT(self->tok_var); Py_VISIT(self->tok_oldval); @@ -1156,7 +1168,7 @@ token_tp_traverse(PyContextToken *self, visitproc visit, void *arg) } static void -token_tp_dealloc(PyContextToken *self) +token_tp_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); (void)token_tp_clear(self); @@ -1164,8 +1176,9 @@ token_tp_dealloc(PyContextToken *self) } static PyObject * -token_tp_repr(PyContextToken *self) +token_tp_repr(PyObject *op) { + PyContextToken *self = _PyContextToken_CAST(op); PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); if (writer == NULL) { return NULL; @@ -1195,14 +1208,16 @@ token_tp_repr(PyContextToken *self) } static PyObject * -token_get_var(PyContextToken *self, void *Py_UNUSED(ignored)) +token_get_var(PyObject *op, void *Py_UNUSED(ignored)) { + PyContextToken *self = _PyContextToken_CAST(op); return Py_NewRef(self->tok_var);; } static PyObject * -token_get_old_value(PyContextToken *self, void *Py_UNUSED(ignored)) +token_get_old_value(PyObject *op, void *Py_UNUSED(ignored)) { + PyContextToken *self = _PyContextToken_CAST(op); if (self->tok_oldval == NULL) { return get_token_missing(); } @@ -1211,8 +1226,8 @@ token_get_old_value(PyContextToken *self, void *Py_UNUSED(ignored)) } static PyGetSetDef PyContextTokenType_getsetlist[] = { - {"var", (getter)token_get_var, NULL, NULL}, - {"old_value", (getter)token_get_old_value, NULL, NULL}, + {"var", token_get_var, NULL, NULL}, + {"old_value", token_get_old_value, NULL, NULL}, {NULL} }; @@ -1228,15 +1243,15 @@ PyTypeObject PyContextToken_Type = { sizeof(PyContextToken), .tp_methods = PyContextTokenType_methods, .tp_getset = PyContextTokenType_getsetlist, - .tp_dealloc = (destructor)token_tp_dealloc, + .tp_dealloc = token_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)token_tp_traverse, - .tp_clear = (inquiry)token_tp_clear, + .tp_traverse = token_tp_traverse, + .tp_clear = token_tp_clear, .tp_new = token_tp_new, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, - .tp_repr = (reprfunc)token_tp_repr, + .tp_repr = token_tp_repr, }; static PyContextToken * @@ -1270,7 +1285,7 @@ context_token_missing_tp_repr(PyObject *self) } static void -context_token_missing_tp_dealloc(_PyContextTokenMissing *Py_UNUSED(self)) +context_token_missing_tp_dealloc(PyObject *Py_UNUSED(self)) { #ifdef Py_DEBUG /* The singleton is statically allocated. */ @@ -1285,7 +1300,7 @@ PyTypeObject _PyContextTokenMissing_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "Token.MISSING", sizeof(_PyContextTokenMissing), - .tp_dealloc = (destructor)context_token_missing_tp_dealloc, + .tp_dealloc = context_token_missing_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_repr = context_token_missing_tp_repr, From 845d924efb2a144120421260e62b9c4c9726fc69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:55:04 +0100 Subject: [PATCH 415/427] gh-111178: fix UBSan failures in `Objects/capsule.c` (GH-128239) fix UBSan failures for `PyCapsule` --- Objects/capsule.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Objects/capsule.c b/Objects/capsule.c index 28965e0f21b7a0..16ae65905ef5ac 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -18,6 +18,8 @@ typedef struct { } PyCapsule; +#define _PyCapsule_CAST(op) ((PyCapsule *)(op)) + static int _is_legal_capsule(PyObject *op, const char *invalid_capsule) @@ -284,7 +286,7 @@ PyCapsule_Import(const char *name, int no_block) static void capsule_dealloc(PyObject *op) { - PyCapsule *capsule = (PyCapsule *)op; + PyCapsule *capsule = _PyCapsule_CAST(op); PyObject_GC_UnTrack(op); if (capsule->destructor) { capsule->destructor(op); @@ -296,7 +298,7 @@ capsule_dealloc(PyObject *op) static PyObject * capsule_repr(PyObject *o) { - PyCapsule *capsule = (PyCapsule *)o; + PyCapsule *capsule = _PyCapsule_CAST(o); const char *name; const char *quote; @@ -314,28 +316,27 @@ capsule_repr(PyObject *o) static int -capsule_traverse(PyCapsule *capsule, visitproc visit, void *arg) +capsule_traverse(PyObject *self, visitproc visit, void *arg) { // Capsule object is only tracked by the GC // if _PyCapsule_SetTraverse() is called, but // this can still be manually triggered by gc.get_referents() - + PyCapsule *capsule = _PyCapsule_CAST(self); if (capsule->traverse_func != NULL) { - return capsule->traverse_func((PyObject*)capsule, visit, arg); + return capsule->traverse_func(self, visit, arg); } - return 0; } static int -capsule_clear(PyCapsule *capsule) +capsule_clear(PyObject *self) { // Capsule object is only tracked by the GC // if _PyCapsule_SetTraverse() is called + PyCapsule *capsule = _PyCapsule_CAST(self); assert(capsule->clear_func != NULL); - - return capsule->clear_func((PyObject*)capsule); + return capsule->clear_func(self); } @@ -358,8 +359,8 @@ PyTypeObject PyCapsule_Type = { .tp_dealloc = capsule_dealloc, .tp_repr = capsule_repr, .tp_doc = PyCapsule_Type__doc__, - .tp_traverse = (traverseproc)capsule_traverse, - .tp_clear = (inquiry)capsule_clear, + .tp_traverse = capsule_traverse, + .tp_clear = capsule_clear, }; From 004f9fd1f22643100aa8163cc9f7bcde7df54973 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 8 Jan 2025 09:00:11 -0800 Subject: [PATCH 416/427] Remove unnecessary LIST_TO_TUPLE conversions (GH-126558) --- Lib/test/test_peepholer.py | 51 ++++++++++++++++++++++++++++++++++++++ Python/flowgraph.c | 7 ++++++ 2 files changed, 58 insertions(+) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index c7da151dce3b37..b5b2b350e77a3b 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1193,5 +1193,56 @@ def get_insts(lno1, lno2, op1, op2): ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + def test_list_to_tuple_get_iter(self): + # for _ in (*foo, *bar) -> for _ in [*foo, *bar] + INTRINSIC_LIST_TO_TUPLE = 6 + insts = [ + ("BUILD_LIST", 0, 1), + ("LOAD_FAST", 0, 2), + ("LIST_EXTEND", 1, 3), + ("LOAD_FAST", 1, 4), + ("LIST_EXTEND", 1, 5), + ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), + ("GET_ITER", None, 7), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 8), + ("STORE_FAST", 2, 9), + ("JUMP", top, 10), + end, + ("END_FOR", None, 11), + ("POP_TOP", None, 12), + ("LOAD_CONST", 0, 13), + ("RETURN_VALUE", None, 14), + ] + expected_insts = [ + ("BUILD_LIST", 0, 1), + ("LOAD_FAST", 0, 2), + ("LIST_EXTEND", 1, 3), + ("LOAD_FAST", 1, 4), + ("LIST_EXTEND", 1, 5), + ("NOP", None, 6), # ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), + ("GET_ITER", None, 7), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 8), + ("STORE_FAST", 2, 9), + ("JUMP", top, 10), + end, + ("END_FOR", None, 11), + ("POP_TOP", None, 12), + ("LOAD_CONST", 0, 13), + ("RETURN_VALUE", None, 14), + ] + self.cfg_optimization_test(insts, expected_insts, consts=[None]) + + def test_list_to_tuple_get_iter_is_safe(self): + a, b = [], [] + for item in (*(items := [0, 1, 2, 3]),): + a.append(item) + b.append(items.pop()) + self.assertEqual(a, [0, 1, 2, 3]) + self.assertEqual(b, [3, 2, 1, 0]) + self.assertEqual(items, []) + + if __name__ == "__main__": unittest.main() diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 017216aadd1f01..24561c1ee04db9 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_flowgraph.h" #include "pycore_compile.h" +#include "pycore_intrinsics.h" #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_opcode_utils.h" @@ -1874,6 +1875,12 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) continue; } break; + case CALL_INTRINSIC_1: + // for _ in (*foo, *bar) -> for _ in [*foo, *bar] + if (oparg == INTRINSIC_LIST_TO_TUPLE && nextop == GET_ITER) { + INSTR_SET_OP0(inst, NOP); + } + break; } } From 34e840f9ddd9c04991cf004e2594c6a1e0e278d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Wed, 8 Jan 2025 20:23:16 +0000 Subject: [PATCH 417/427] GH-66409: check if exec_prefix is the same as prefix before searching executable_dir (#127974) --- .../2024-12-15-21-11-26.gh-issue-66409.wv109z.rst | 3 +++ Modules/getpath.py | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-21-11-26.gh-issue-66409.wv109z.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-21-11-26.gh-issue-66409.wv109z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-21-11-26.gh-issue-66409.wv109z.rst new file mode 100644 index 00000000000000..0d70ad06c97968 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-21-11-26.gh-issue-66409.wv109z.rst @@ -0,0 +1,3 @@ +During the :ref:`path initialization `, we now check if +``base_exec_prefix`` is the same as ``base_prefix`` before falling back to +searching the Python interpreter directory. diff --git a/Modules/getpath.py b/Modules/getpath.py index c34101e720851d..be2210345afbda 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -625,6 +625,8 @@ def search_up(prefix, *landmarks, test=isfile): # gh-100320: Our PYDs are assumed to be relative to the Lib directory # (that is, prefix) rather than the executable (that is, executable_dir) exec_prefix = prefix + if not exec_prefix and prefix and isdir(joinpath(prefix, PLATSTDLIB_LANDMARK)): + exec_prefix = prefix if not exec_prefix and executable_dir: exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) if not exec_prefix and EXEC_PREFIX: From 58a9133fc2caea15d11116f0b5bd6832374cb88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Wed, 8 Jan 2025 21:10:43 +0000 Subject: [PATCH 418/427] acks: add myself (#128652) --- Misc/ACKS | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/ACKS b/Misc/ACKS index d7585c16c8169c..deda334bee7417 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1038,6 +1038,7 @@ Erno Kuusela Kabir Kwatra Ross Lagerwall Cameron Laird +Filipe Laíns Loïc Lajeanne Alexander Lakeev David Lam From a1284e97979ff73ad72ad06c796b904137950576 Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Thu, 9 Jan 2025 00:38:02 +0100 Subject: [PATCH 419/427] gh-97850: Remove the mention of removal from `ResourceReader` docs (#128602) Remove the mention of removal from ResourceReader docs --- Doc/library/importlib.resources.abc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 54995ddbfbca12..4085bdf6598d98 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -43,7 +43,7 @@ :const:`None`. An object compatible with this ABC should only be returned when the specified module is a package. - .. deprecated-removed:: 3.12 3.14 + .. deprecated:: 3.12 Use :class:`importlib.resources.abc.TraversableResources` instead. .. abstractmethod:: open_resource(resource) From 4685401845ba3e2ab8c9f4a9a10aa2969b11985f Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 9 Jan 2025 14:49:05 +0530 Subject: [PATCH 420/427] gh-79149: document reentrant safety of `loop.call_soon_threadsafe` (#128662) Co-authored-by: Thomas Grainger --- Doc/library/asyncio-eventloop.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index ccb362d8c31ddf..bfc0d16f023e5e 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -246,6 +246,9 @@ Scheduling callbacks another thread, this function *must* be used, since :meth:`call_soon` is not thread-safe. + This function is safe to be called from a reentrant context or signal handler, + however, it is not safe or fruitful to use the returned handle in such contexts. + Raises :exc:`RuntimeError` if called on a loop that's been closed. This can happen on a secondary thread when the main application is shutting down. From 1439b81928f1b52c5a0ac7fd81fdd66afd5f72da Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 9 Jan 2025 11:10:28 +0100 Subject: [PATCH 421/427] gh-128629: Add Py_PACK_VERSION and Py_PACK_FULL_VERSION (GH-128630) --- Doc/c-api/apiabiversion.rst | 102 +++++++++++++----- Doc/data/stable_abi.dat | 2 + Doc/whatsnew/3.14.rst | 4 + Include/patchlevel.h | 26 +++-- Include/pymacro.h | 9 ++ Lib/test/test_capi/test_misc.py | 43 ++++++++ Lib/test/test_stable_abi_ctypes.py | 2 + ...-01-08-13-13-18.gh-issue-128629.gSmzyl.rst | 2 + Misc/stable_abi.toml | 4 + Modules/Setup.stdlib.in | 2 +- Modules/_testlimitedcapi.c | 3 + Modules/_testlimitedcapi/clinic/version.c.h | 93 ++++++++++++++++ Modules/_testlimitedcapi/parts.h | 1 + Modules/_testlimitedcapi/version.c | 77 +++++++++++++ PC/python3dll.c | 2 + PCbuild/_testlimitedcapi.vcxproj | 1 + PCbuild/_testlimitedcapi.vcxproj.filters | 1 + Python/modsupport.c | 17 +++ 18 files changed, 358 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-01-08-13-13-18.gh-issue-128629.gSmzyl.rst create mode 100644 Modules/_testlimitedcapi/clinic/version.c.h create mode 100644 Modules/_testlimitedcapi/version.c diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index f6c8284daeacb0..96050f59bd5250 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -6,9 +6,13 @@ API and ABI Versioning *********************** + +Build-time version constants +---------------------------- + CPython exposes its version number in the following macros. -Note that these correspond to the version code is **built** with, -not necessarily the version used at **run time**. +Note that these correspond to the version code is **built** with. +See :c:var:`Py_Version` for the version used at **run time**. See :ref:`stable` for a discussion of API and ABI stability across versions. @@ -37,37 +41,83 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. .. c:macro:: PY_VERSION_HEX The Python version number encoded in a single integer. + See :c:func:`Py_PACK_FULL_VERSION` for the encoding details. - The underlying version information can be found by treating it as a 32 bit - number in the following manner: - - +-------+-------------------------+-------------------------+--------------------------+ - | Bytes | Bits (big endian order) | Meaning | Value for ``3.4.1a2`` | - +=======+=========================+=========================+==========================+ - | 1 | 1-8 | ``PY_MAJOR_VERSION`` | ``0x03`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 2 | 9-16 | ``PY_MINOR_VERSION`` | ``0x04`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 3 | 17-24 | ``PY_MICRO_VERSION`` | ``0x01`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 4 | 25-28 | ``PY_RELEASE_LEVEL`` | ``0xA`` | - + +-------------------------+-------------------------+--------------------------+ - | | 29-32 | ``PY_RELEASE_SERIAL`` | ``0x2`` | - +-------+-------------------------+-------------------------+--------------------------+ + Use this for numeric comparisons, for example, + ``#if PY_VERSION_HEX >= ...``. - Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is - hexversion ``0x030a00f0``. - Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. - - This version is also available via the symbol :c:var:`Py_Version`. +Run-time version +---------------- .. c:var:: const unsigned long Py_Version - The Python runtime version number encoded in a single constant integer, with - the same format as the :c:macro:`PY_VERSION_HEX` macro. + The Python runtime version number encoded in a single constant integer. + See :c:func:`Py_PACK_FULL_VERSION` for the encoding details. This contains the Python version used at run time. + Use this for numeric comparisons, for example, ``if (Py_Version >= ...)``. + .. versionadded:: 3.11 -All the given macros are defined in :source:`Include/patchlevel.h`. + +Bit-packing macros +------------------ + +.. c:function:: uint32_t Py_PACK_FULL_VERSION(int major, int minor, int micro, int release_level, int release_serial) + + Return the given version, encoded as a single 32-bit integer with + the following structure: + + +------------------+-------+----------------+-----------+--------------------------+ + | | No. | | | Example values | + | | of | | +-------------+------------+ + | Argument | bits | Bit mask | Bit shift | ``3.4.1a2`` | ``3.10.0`` | + +==================+=======+================+===========+=============+============+ + | *major* | 8 | ``0xFF000000`` | 24 | ``0x03`` | ``0x03`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *minor* | 8 | ``0x00FF0000`` | 16 | ``0x04`` | ``0x0A`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *micro* | 8 | ``0x0000FF00`` | 8 | ``0x01`` | ``0x00`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *release_level* | 4 | ``0x000000F0`` | 4 | ``0xA`` | ``0xF`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *release_serial* | 4 | ``0x0000000F`` | 0 | ``0x2`` | ``0x0`` | + +------------------+-------+----------------+-----------+-------------+------------+ + + For example: + + +-------------+------------------------------------+-----------------+ + | Version | ``Py_PACK_FULL_VERSION`` arguments | Encoded version | + +=============+====================================+=================+ + | ``3.4.1a2`` | ``(3, 4, 1, 0xA, 2)`` | ``0x030401a2`` | + +-------------+------------------------------------+-----------------+ + | ``3.10.0`` | ``(3, 10, 0, 0xF, 0)`` | ``0x030a00f0`` | + +-------------+------------------------------------+-----------------+ + + Out-of range bits in the arguments are ignored. + That is, the macro can be defined as: + + .. code-block:: c + + #ifndef Py_PACK_FULL_VERSION + #define Py_PACK_FULL_VERSION(X, Y, Z, LEVEL, SERIAL) ( \ + (((X) & 0xff) << 24) | \ + (((Y) & 0xff) << 16) | \ + (((Z) & 0xff) << 8) | \ + (((LEVEL) & 0xf) << 4) | \ + (((SERIAL) & 0xf) << 0)) + #endif + + ``Py_PACK_FULL_VERSION`` is primarily a macro, intended for use in + ``#if`` directives, but it is also available as an exported function. + + .. versionadded:: 3.14 + +.. c:function:: uint32_t Py_PACK_VERSION(int major, int minor) + + Equivalent to ``Py_PACK_FULL_VERSION(major, minor, 0, 0, 0)``. + The result does not correspond to any Python release, but is useful + in numeric comparisons. + + .. versionadded:: 3.14 diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 6f9d27297e8f65..c15f82603aa944 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -883,6 +883,8 @@ func,Py_Main,3.2,, func,Py_MakePendingCalls,3.2,, func,Py_NewInterpreter,3.2,, func,Py_NewRef,3.10,, +func,Py_PACK_FULL_VERSION,3.14,, +func,Py_PACK_VERSION,3.14,, func,Py_REFCNT,3.14,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 16851b4e63ea2c..72abfebd46f2b9 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1243,6 +1243,10 @@ New features file. (Contributed by Victor Stinner in :gh:`127350`.) +* Add macros :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION` for + bit-packing Python version numbers. + (Contributed by Petr Viktorin in :gh:`128629`.) + Porting to Python 3.14 ---------------------- diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 6d4f719fcde5a8..eca2ca08a4337c 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -1,4 +1,5 @@ - +#ifndef _Py_PATCHLEVEL_H +#define _Py_PATCHLEVEL_H /* Python version identification scheme. When the major or minor version changes, the VERSION variable in @@ -26,10 +27,23 @@ #define PY_VERSION "3.14.0a3+" /*--end constants--*/ + +#define _Py_PACK_FULL_VERSION(X, Y, Z, LEVEL, SERIAL) ( \ + (((X) & 0xff) << 24) | \ + (((Y) & 0xff) << 16) | \ + (((Z) & 0xff) << 8) | \ + (((LEVEL) & 0xf) << 4) | \ + (((SERIAL) & 0xf) << 0)) + /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. Use this for numeric comparisons, e.g. #if PY_VERSION_HEX >= ... */ -#define PY_VERSION_HEX ((PY_MAJOR_VERSION << 24) | \ - (PY_MINOR_VERSION << 16) | \ - (PY_MICRO_VERSION << 8) | \ - (PY_RELEASE_LEVEL << 4) | \ - (PY_RELEASE_SERIAL << 0)) +#define PY_VERSION_HEX _Py_PACK_FULL_VERSION( \ + PY_MAJOR_VERSION, \ + PY_MINOR_VERSION, \ + PY_MICRO_VERSION, \ + PY_RELEASE_LEVEL, \ + PY_RELEASE_SERIAL) + +// Public Py_PACK_VERSION is declared in pymacro.h; it needs . + +#endif //_Py_PATCHLEVEL_H diff --git a/Include/pymacro.h b/Include/pymacro.h index e0378f9d27a048..a82f347866e8d0 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -190,4 +190,13 @@ // "comparison of unsigned expression in '< 0' is always false". #define _Py_IS_TYPE_SIGNED(type) ((type)(-1) <= 0) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 // 3.14 +// Version helpers. These are primarily macros, but have exported equivalents. +PyAPI_FUNC(uint32_t) Py_PACK_FULL_VERSION(int x, int y, int z, int level, int serial); +PyAPI_FUNC(uint32_t) Py_PACK_VERSION(int x, int y); +#define Py_PACK_FULL_VERSION _Py_PACK_FULL_VERSION +#define Py_PACK_VERSION(X, Y) Py_PACK_FULL_VERSION(X, Y, 0, 0, 0) +#endif // Py_LIMITED_API < 3.14 + + #endif /* Py_PYMACRO_H */ diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index ada30181aeeca9..b62bc4c2ecd980 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -3335,6 +3335,49 @@ def run(self): self.assertEqual(len(set(py_thread_ids)), len(py_thread_ids), py_thread_ids) +class TestVersions(unittest.TestCase): + full_cases = ( + (3, 4, 1, 0xA, 2, 0x030401a2), + (3, 10, 0, 0xF, 0, 0x030a00f0), + (0x103, 0x10B, 0xFF00, -1, 0xF0, 0x030b00f0), # test masking + ) + xy_cases = ( + (3, 4, 0x03040000), + (3, 10, 0x030a0000), + (0x103, 0x10B, 0x030b0000), # test masking + ) + + def test_pack_full_version(self): + for *args, expected in self.full_cases: + with self.subTest(hexversion=hex(expected)): + result = _testlimitedcapi.pack_full_version(*args) + self.assertEqual(result, expected) + + def test_pack_version(self): + for *args, expected in self.xy_cases: + with self.subTest(hexversion=hex(expected)): + result = _testlimitedcapi.pack_version(*args) + self.assertEqual(result, expected) + + def test_pack_full_version_ctypes(self): + ctypes = import_helper.import_module('ctypes') + ctypes_func = ctypes.pythonapi.Py_PACK_FULL_VERSION + ctypes_func.restype = ctypes.c_uint32 + ctypes_func.argtypes = [ctypes.c_int] * 5 + for *args, expected in self.full_cases: + with self.subTest(hexversion=hex(expected)): + result = ctypes_func(*args) + self.assertEqual(result, expected) + + def test_pack_version_ctypes(self): + ctypes = import_helper.import_module('ctypes') + ctypes_func = ctypes.pythonapi.Py_PACK_VERSION + ctypes_func.restype = ctypes.c_uint32 + ctypes_func.argtypes = [ctypes.c_int] * 2 + for *args, expected in self.xy_cases: + with self.subTest(hexversion=hex(expected)): + result = ctypes_func(*args) + self.assertEqual(result, expected) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index fa08dc6a25b0ea..f3724ce6d4d15a 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -901,6 +901,8 @@ def test_windows_feature_macros(self): "Py_MakePendingCalls", "Py_NewInterpreter", "Py_NewRef", + "Py_PACK_FULL_VERSION", + "Py_PACK_VERSION", "Py_REFCNT", "Py_ReprEnter", "Py_ReprLeave", diff --git a/Misc/NEWS.d/next/C_API/2025-01-08-13-13-18.gh-issue-128629.gSmzyl.rst b/Misc/NEWS.d/next/C_API/2025-01-08-13-13-18.gh-issue-128629.gSmzyl.rst new file mode 100644 index 00000000000000..cde5bf38f754b6 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-01-08-13-13-18.gh-issue-128629.gSmzyl.rst @@ -0,0 +1,2 @@ +Add macros :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION` for +bit-packing Python version numbers. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index f9e51f0683c965..276526a1b6908e 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2540,3 +2540,7 @@ added = '3.14' [function.PyType_Freeze] added = '3.14' +[function.Py_PACK_FULL_VERSION] + added = '3.14' +[function.Py_PACK_VERSION] + added = '3.14' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 52c0f883d383db..b7357f41768a2f 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index ba83a23117b2a5..bcc69a339ec5c4 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -83,5 +83,8 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_VectorcallLimited(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Version(mod) < 0) { + return NULL; + } return mod; } diff --git a/Modules/_testlimitedcapi/clinic/version.c.h b/Modules/_testlimitedcapi/clinic/version.c.h new file mode 100644 index 00000000000000..096c7dd528b332 --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/version.c.h @@ -0,0 +1,93 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_testlimitedcapi_pack_full_version__doc__, +"pack_full_version($module, major, minor, micro, level, serial, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_PACK_FULL_VERSION_METHODDEF \ + {"pack_full_version", (PyCFunction)(void(*)(void))_testlimitedcapi_pack_full_version, METH_FASTCALL, _testlimitedcapi_pack_full_version__doc__}, + +static PyObject * +_testlimitedcapi_pack_full_version_impl(PyObject *module, int major, + int minor, int micro, int level, + int serial); + +static PyObject * +_testlimitedcapi_pack_full_version(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int major; + int minor; + int micro; + int level; + int serial; + + if (nargs != 5) { + PyErr_Format(PyExc_TypeError, "pack_full_version expected 5 arguments, got %zd", nargs); + goto exit; + } + major = PyLong_AsInt(args[0]); + if (major == -1 && PyErr_Occurred()) { + goto exit; + } + minor = PyLong_AsInt(args[1]); + if (minor == -1 && PyErr_Occurred()) { + goto exit; + } + micro = PyLong_AsInt(args[2]); + if (micro == -1 && PyErr_Occurred()) { + goto exit; + } + level = PyLong_AsInt(args[3]); + if (level == -1 && PyErr_Occurred()) { + goto exit; + } + serial = PyLong_AsInt(args[4]); + if (serial == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _testlimitedcapi_pack_full_version_impl(module, major, minor, micro, level, serial); + +exit: + return return_value; +} + +PyDoc_STRVAR(_testlimitedcapi_pack_version__doc__, +"pack_version($module, major, minor, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_PACK_VERSION_METHODDEF \ + {"pack_version", (PyCFunction)(void(*)(void))_testlimitedcapi_pack_version, METH_FASTCALL, _testlimitedcapi_pack_version__doc__}, + +static PyObject * +_testlimitedcapi_pack_version_impl(PyObject *module, int major, int minor); + +static PyObject * +_testlimitedcapi_pack_version(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int major; + int minor; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "pack_version expected 2 arguments, got %zd", nargs); + goto exit; + } + major = PyLong_AsInt(args[0]); + if (major == -1 && PyErr_Occurred()) { + goto exit; + } + minor = PyLong_AsInt(args[1]); + if (minor == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _testlimitedcapi_pack_version_impl(module, major, minor); + +exit: + return return_value; +} +/*[clinic end generated code: output=aed3e226da77f2d2 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 4107b150c5b4e0..56d566b66565a3 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -40,5 +40,6 @@ int _PyTestLimitedCAPI_Init_Sys(PyObject *module); int _PyTestLimitedCAPI_Init_Tuple(PyObject *module); int _PyTestLimitedCAPI_Init_Unicode(PyObject *module); int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); +int _PyTestLimitedCAPI_Init_Version(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testlimitedcapi/version.c b/Modules/_testlimitedcapi/version.c new file mode 100644 index 00000000000000..57cd6e4e928ea3 --- /dev/null +++ b/Modules/_testlimitedcapi/version.c @@ -0,0 +1,77 @@ +/* Test version macros in the limited API */ + +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030e0000 // Added in 3.14 +#endif + +#include "parts.h" +#include "clinic/version.c.h" +#include + +/*[clinic input] +module _testlimitedcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2700057f9c1135ba]*/ + +/*[clinic input] +_testlimitedcapi.pack_full_version + + major: int + minor: int + micro: int + level: int + serial: int + / +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_pack_full_version_impl(PyObject *module, int major, + int minor, int micro, int level, + int serial) +/*[clinic end generated code: output=b87a1e9805648861 input=2a304423be61d2ac]*/ +{ + uint32_t macro_result = Py_PACK_FULL_VERSION( + major, minor, micro, level, serial); +#undef Py_PACK_FULL_VERSION + uint32_t func_result = Py_PACK_FULL_VERSION( + major, minor, micro, level, serial); + + assert(macro_result == func_result); + return PyLong_FromUnsignedLong((unsigned long)func_result); +} + +/*[clinic input] +_testlimitedcapi.pack_version + + major: int + minor: int + / +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_pack_version_impl(PyObject *module, int major, int minor) +/*[clinic end generated code: output=771247bbd06e7883 input=3e39e9dcbc09e86a]*/ +{ + uint32_t macro_result = Py_PACK_VERSION(major, minor); +#undef Py_PACK_VERSION + uint32_t func_result = Py_PACK_VERSION(major, minor); + + assert(macro_result == func_result); + return PyLong_FromUnsignedLong((unsigned long)func_result); +} + +static PyMethodDef TestMethods[] = { + _TESTLIMITEDCAPI_PACK_FULL_VERSION_METHODDEF + _TESTLIMITEDCAPI_PACK_VERSION_METHODDEF + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Version(PyObject *m) +{ + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + return 0; +} diff --git a/PC/python3dll.c b/PC/python3dll.c index 8657ddb9fa5155..84b3c735240b73 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -81,6 +81,8 @@ EXPORT_FUNC(Py_Main) EXPORT_FUNC(Py_MakePendingCalls) EXPORT_FUNC(Py_NewInterpreter) EXPORT_FUNC(Py_NewRef) +EXPORT_FUNC(Py_PACK_FULL_VERSION) +EXPORT_FUNC(Py_PACK_VERSION) EXPORT_FUNC(Py_REFCNT) EXPORT_FUNC(Py_ReprEnter) EXPORT_FUNC(Py_ReprLeave) diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 846e027e10c7fa..0ea5edba3aa9a7 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -112,6 +112,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 57be2e2fc5b950..b379090eb599f5 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -28,6 +28,7 @@ + diff --git a/Python/modsupport.c b/Python/modsupport.c index 0fb7783345c78e..517dc971f88c87 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -648,3 +648,20 @@ PyModule_AddType(PyObject *module, PyTypeObject *type) return PyModule_AddObjectRef(module, name, (PyObject *)type); } + + +/* Exported functions for version helper macros */ + +#undef Py_PACK_FULL_VERSION +uint32_t +Py_PACK_FULL_VERSION(int x, int y, int z, int level, int serial) +{ + return _Py_PACK_FULL_VERSION(x, y, z, level, serial); +} + +#undef Py_PACK_VERSION +uint32_t +Py_PACK_VERSION(int x, int y) +{ + return Py_PACK_FULL_VERSION(x, y, 0, 0, 0); +} From b2adf556747d080f04b53ba4063b627c2dbe41d1 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 9 Jan 2025 15:40:45 +0530 Subject: [PATCH 422/427] gh-126137: improve docs for `loop.add_reader` and `loop.add_writer` (#128666) --- Doc/library/asyncio-eventloop.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index bfc0d16f023e5e..072ab206f25e4f 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -970,6 +970,9 @@ Watching file descriptors invoke *callback* with the specified arguments once *fd* is available for reading. + Any preexisting callback registered for *fd* is cancelled and replaced by + *callback*. + .. method:: loop.remove_reader(fd) Stop monitoring the *fd* file descriptor for read availability. Returns @@ -981,6 +984,9 @@ Watching file descriptors invoke *callback* with the specified arguments once *fd* is available for writing. + Any preexisting callback registered for *fd* is cancelled and replaced by + *callback*. + Use :func:`functools.partial` :ref:`to pass keyword arguments ` to *callback*. From 4322a318ea98ceeb95d88b7ae6b5cfa3572d2069 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 9 Jan 2025 17:50:12 +0530 Subject: [PATCH 423/427] gh-124433: fix docs for `asyncio.Queue.task_done` (#128669) --- Doc/library/asyncio-queue.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index 61991bf2f4ed1d..066edd424d150e 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -115,11 +115,11 @@ Queue .. method:: task_done() - Indicate that a formerly enqueued task is complete. + Indicate that a formerly enqueued work item is complete. Used by queue consumers. For each :meth:`~Queue.get` used to - fetch a task, a subsequent call to :meth:`task_done` tells the - queue that the processing on the task is complete. + fetch a work item, a subsequent call to :meth:`task_done` tells the + queue that the processing on the work item is complete. If a :meth:`join` is currently blocking, it will resume when all items have been processed (meaning that a :meth:`task_done` From ea39c8b08d8f025273bfa5b7a95f7b5984dc1e86 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 9 Jan 2025 08:54:44 -0500 Subject: [PATCH 424/427] gh-118915: Document `PyUnstable_InterpreterState_GetMainModule` (GH-128483) --- Doc/c-api/init.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index dd63dd013e32dc..97996a6f69dd22 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1492,6 +1492,17 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.8 + +.. c:function:: PyObject* PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) + + Return a :term:`strong reference` to the ``__main__`` `module object `_ + for the given interpreter. + + The caller must hold the GIL. + + .. versionadded:: 3.13 + + .. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) Type of a frame evaluation function. From 43ac9f505903ba806aa6a5d93e6a67beb04bebc4 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jan 2025 17:25:03 +0300 Subject: [PATCH 425/427] gh-128673: Increase coverage of `typing.get_type_hints` (#128674) --- Lib/test/test_typing.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 45ba7611059e43..1c86b95e8e5c29 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7152,6 +7152,25 @@ class C: self.assertEqual(get_type_hints(C, format=annotationlib.Format.STRING), {'x': 'undefined'}) + def test_get_type_hints_format_function(self): + def func(x: undefined) -> undefined: ... + + # VALUE + with self.assertRaises(NameError): + get_type_hints(func) + with self.assertRaises(NameError): + get_type_hints(func, format=annotationlib.Format.VALUE) + + # FORWARDREF + self.assertEqual( + get_type_hints(func, format=annotationlib.Format.FORWARDREF), + {'x': ForwardRef('undefined'), 'return': ForwardRef('undefined')}, + ) + + # STRING + self.assertEqual(get_type_hints(func, format=annotationlib.Format.STRING), + {'x': 'undefined', 'return': 'undefined'}) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): From b725297cee9e5608b709f3c7291d974c97f68fff Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jan 2025 18:15:13 +0300 Subject: [PATCH 426/427] gh-128661: Fix `typing.evaluate_forward_ref` not showing deprecation (#128663) gh-128661: Fix `typing.evaluate_forward_ref` not showing deprecataion --- Lib/test/test_typing.py | 46 +++++++++++++++++++ Lib/typing.py | 2 +- ...-01-09-12-06-52.gh-issue-128661.ixx_0z.rst | 2 + 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-09-12-06-52.gh-issue-128661.ixx_0z.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 1c86b95e8e5c29..c51ee763890af2 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -45,6 +45,7 @@ import textwrap import typing import weakref +import warnings import types from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper, run_code @@ -7273,6 +7274,51 @@ class C(Generic[T]): pass self.assertEqual(get_args(Unpack[tuple[Unpack[Ts]]]), (tuple[Unpack[Ts]],)) +class EvaluateForwardRefTests(BaseTestCase): + def test_evaluate_forward_ref(self): + int_ref = ForwardRef('int') + missing = ForwardRef('missing') + self.assertIs( + typing.evaluate_forward_ref(int_ref, type_params=()), + int, + ) + self.assertIs( + typing.evaluate_forward_ref( + int_ref, type_params=(), format=annotationlib.Format.FORWARDREF, + ), + int, + ) + self.assertIs( + typing.evaluate_forward_ref( + missing, type_params=(), format=annotationlib.Format.FORWARDREF, + ), + missing, + ) + self.assertEqual( + typing.evaluate_forward_ref( + int_ref, type_params=(), format=annotationlib.Format.STRING, + ), + 'int', + ) + + def test_evaluate_forward_ref_no_type_params(self): + ref = ForwardRef('int') + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing.evaluate_forward_ref' is deprecated, " + "as it leads to incorrect behaviour" + ), + ): + typing.evaluate_forward_ref(ref) + + # No warnings when `type_params` is passed: + with warnings.catch_warnings(record=True) as w: + typing.evaluate_forward_ref(ref, type_params=()) + self.assertEqual(w, []) + + class CollectionsAbcTests(BaseTestCase): def test_hashable(self): diff --git a/Lib/typing.py b/Lib/typing.py index e69b485422cbd2..66570db7a5bd74 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1024,7 +1024,7 @@ def evaluate_forward_ref( owner=None, globals=None, locals=None, - type_params=None, + type_params=_sentinel, format=annotationlib.Format.VALUE, _recursive_guard=frozenset(), ): diff --git a/Misc/NEWS.d/next/Library/2025-01-09-12-06-52.gh-issue-128661.ixx_0z.rst b/Misc/NEWS.d/next/Library/2025-01-09-12-06-52.gh-issue-128661.ixx_0z.rst new file mode 100644 index 00000000000000..6c52b3dcc0ed00 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-09-12-06-52.gh-issue-128661.ixx_0z.rst @@ -0,0 +1,2 @@ +Fixes :func:`typing.evaluate_forward_ref` not showing deprecation when +``type_params`` arg is not passed. From 7dc41ad6a7826ffc675f088972de96624917696e Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 9 Jan 2025 21:26:00 +0530 Subject: [PATCH 427/427] gh-128002: fix `asyncio.all_tasks` against concurrent deallocations of tasks (#128541) --- Include/internal/pycore_object.h | 2 +- Modules/_asynciomodule.c | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index d7d68f938a9f0a..e26cb7673f939c 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -120,7 +120,7 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( PyAPI_DATA(Py_ssize_t) _Py_RefTotal; extern void _Py_AddRefTotal(PyThreadState *, Py_ssize_t); -extern void _Py_IncRefTotal(PyThreadState *); +extern PyAPI_FUNC(void) _Py_IncRefTotal(PyThreadState *); extern void _Py_DecRefTotal(PyThreadState *); # define _Py_DEC_REFTOTAL(interp) \ diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index b8b184af04a7cb..48f0ef95934fa4 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3772,11 +3772,20 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) llist_for_each_safe(node, &state->asyncio_tasks_head) { TaskObj *task = llist_data(node, TaskObj, task_node); - if (PyList_Append(tasks, (PyObject *)task) < 0) { - Py_DECREF(tasks); - Py_DECREF(loop); - err = 1; - break; + // The linked list holds borrowed references to task + // as such it is possible that the task is concurrently + // deallocated while added to this list. + // To protect against concurrent deallocations, + // we first try to incref the task which would fail + // if it is concurrently getting deallocated in another thread, + // otherwise it gets added to the list. + if (_Py_TryIncref((PyObject *)task)) { + if (_PyList_AppendTakeRef((PyListObject *)tasks, (PyObject *)task) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + err = 1; + break; + } } } ASYNCIO_STATE_UNLOCK(state);