From 35ae4aab1aae93c1c11c45ac431787ff79ce7907 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 6 Aug 2024 06:56:50 +0300 Subject: [PATCH 1/6] gh-122686: Pin attrs package (for Hypothesis workflow) (#122687) Co-authored-by: Victor Stinner --- Tools/requirements-hypothesis.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index ab3f39ac6ee087..03f955ba8bf310 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,7 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. +# see https://github.com/python/cpython/issues/122686 +attrs<=23.2.0 + hypothesis==6.104.2 From b0c48b8fd88f26b31ec2f743358091073277dcde Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 6 Aug 2024 05:28:58 +0100 Subject: [PATCH 2/6] gh-116622: Android logging fixes (#122698) Modifies the handling of stdout/stderr redirection on Android to accomodate the rate and buffer size limits imposed by Android's logging infrastructure. --- Lib/_android_support.py | 128 +++++++++++++---- Lib/test/test_android.py | 132 +++++++++++++++--- ...-08-05-19-04-06.gh-issue-116622.3LWUzE.rst | 1 + 3 files changed, 213 insertions(+), 48 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-08-05-19-04-06.gh-issue-116622.3LWUzE.rst diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 590e85ea8c2db1..d5d13ec6a48e14 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -1,19 +1,20 @@ import io import sys - +from threading import RLock +from time import sleep, time # The maximum length of a log message in bytes, including the level marker and -# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in -# platform/system/logging/liblog/include/log/log.h. As of API level 30, messages -# longer than this will be be truncated by logcat. This limit has already been -# reduced at least once in the history of Android (from 4076 to 4068 between API -# level 23 and 26), so leave some headroom. +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71. +# Messages longer than this will be be truncated by logcat. This limit has already +# been reduced at least once in the history of Android (from 4076 to 4068 between +# API level 23 and 26), so leave some headroom. MAX_BYTES_PER_WRITE = 4000 # UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this -# size ensures that TextIOWrapper can always avoid exceeding MAX_BYTES_PER_WRITE. +# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE. # However, if the actual number of bytes per character is smaller than that, -# then TextIOWrapper may still join multiple consecutive text writes into binary +# then we may still join multiple consecutive text writes into binary # writes containing a larger number of characters. MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 @@ -26,18 +27,22 @@ def init_streams(android_log_write, stdout_prio, stderr_prio): if sys.executable: return # Not embedded in an app. + global logcat + logcat = Logcat(android_log_write) + sys.stdout = TextLogStream( - android_log_write, stdout_prio, "python.stdout", errors=sys.stdout.errors) + stdout_prio, "python.stdout", errors=sys.stdout.errors) sys.stderr = TextLogStream( - android_log_write, stderr_prio, "python.stderr", errors=sys.stderr.errors) + stderr_prio, "python.stderr", errors=sys.stderr.errors) class TextLogStream(io.TextIOWrapper): - def __init__(self, android_log_write, prio, tag, **kwargs): + def __init__(self, prio, tag, **kwargs): kwargs.setdefault("encoding", "UTF-8") - kwargs.setdefault("line_buffering", True) - super().__init__(BinaryLogStream(android_log_write, prio, tag), **kwargs) - self._CHUNK_SIZE = MAX_BYTES_PER_WRITE + super().__init__(BinaryLogStream(prio, tag), **kwargs) + self._lock = RLock() + self._pending_bytes = [] + self._pending_bytes_count = 0 def __repr__(self): return f"" @@ -52,19 +57,48 @@ def write(self, s): s = str.__str__(s) # We want to emit one log message per line wherever possible, so split - # the string before sending it to the superclass. Note that - # "".splitlines() == [], so nothing will be logged for an empty string. - for line in s.splitlines(keepends=True): - while line: - super().write(line[:MAX_CHARS_PER_WRITE]) - line = line[MAX_CHARS_PER_WRITE:] + # the string into lines first. Note that "".splitlines() == [], so + # nothing will be logged for an empty string. + with self._lock: + for line in s.splitlines(keepends=True): + while line: + chunk = line[:MAX_CHARS_PER_WRITE] + line = line[MAX_CHARS_PER_WRITE:] + self._write_chunk(chunk) return len(s) + # The size and behavior of TextIOWrapper's buffer is not part of its public + # API, so we handle buffering ourselves to avoid truncation. + def _write_chunk(self, s): + b = s.encode(self.encoding, self.errors) + if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE: + self.flush() + + self._pending_bytes.append(b) + self._pending_bytes_count += len(b) + if ( + self.write_through + or b.endswith(b"\n") + or self._pending_bytes_count > MAX_BYTES_PER_WRITE + ): + self.flush() + + def flush(self): + with self._lock: + self.buffer.write(b"".join(self._pending_bytes)) + self._pending_bytes.clear() + self._pending_bytes_count = 0 + + # Since this is a line-based logging system, line buffering cannot be turned + # off, i.e. a newline always causes a flush. + @property + def line_buffering(self): + return True + class BinaryLogStream(io.RawIOBase): - def __init__(self, android_log_write, prio, tag): - self.android_log_write = android_log_write + def __init__(self, prio, tag): self.prio = prio self.tag = tag @@ -85,10 +119,48 @@ def write(self, b): # Writing an empty string to the stream should have no effect. if b: - # Encode null bytes using "modified UTF-8" to avoid truncating the - # message. This should not affect the return value, as the caller - # may be expecting it to match the length of the input. - self.android_log_write(self.prio, self.tag, - b.replace(b"\x00", b"\xc0\x80")) - + logcat.write(self.prio, self.tag, b) return len(b) + + +# When a large volume of data is written to logcat at once, e.g. when a test +# module fails in --verbose3 mode, there's a risk of overflowing logcat's own +# buffer and losing messages. We avoid this by imposing a rate limit using the +# token bucket algorithm, based on a conservative estimate of how fast `adb +# logcat` can consume data. +MAX_BYTES_PER_SECOND = 1024 * 1024 + +# The logcat buffer size of a device can be determined by running `logcat -g`. +# We set the token bucket size to half of the buffer size of our current minimum +# API level, because other things on the system will be producing messages as +# well. +BUCKET_SIZE = 128 * 1024 + +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 +PER_MESSAGE_OVERHEAD = 28 + + +class Logcat: + def __init__(self, android_log_write): + self.android_log_write = android_log_write + self._lock = RLock() + self._bucket_level = 0 + self._prev_write_time = time() + + def write(self, prio, tag, message): + # Encode null bytes using "modified UTF-8" to avoid them truncating the + # message. + message = message.replace(b"\x00", b"\xc0\x80") + + with self._lock: + now = time() + self._bucket_level += ( + (now - self._prev_write_time) * MAX_BYTES_PER_SECOND) + self._bucket_level = min(self._bucket_level, BUCKET_SIZE) + self._prev_write_time = now + + self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message) + if self._bucket_level < 0: + sleep(-self._bucket_level / MAX_BYTES_PER_SECOND) + + self.android_log_write(prio, tag, message) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 115882a4c281f6..82035061bb6fdd 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -1,14 +1,17 @@ +import io import platform import queue import re import subprocess import sys import unittest +from _android_support import TextLogStream from array import array -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager from threading import Thread from test.support import LOOPBACK_TIMEOUT -from time import time +from time import sleep, time +from unittest.mock import patch if sys.platform != "android": @@ -81,18 +84,39 @@ def unbuffered(self, stream): finally: stream.reconfigure(write_through=False) + # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't + # test them directly. Detect this mode and use some temporary streams with + # the same properties. + def stream_context(self, stream_name, level): + # https://developer.android.com/ndk/reference/group/logging + prio = {"I": 4, "W": 5}[level] + + stack = ExitStack() + stack.enter_context(self.subTest(stream_name)) + stream = getattr(sys, stream_name) + if isinstance(stream, io.StringIO): + stack.enter_context( + patch( + f"sys.{stream_name}", + TextLogStream( + prio, f"python.{stream_name}", errors="backslashreplace" + ), + ) + ) + return stack + def test_str(self): for stream_name, level in [("stdout", "I"), ("stderr", "W")]: - with self.subTest(stream=stream_name): + with self.stream_context(stream_name, level): stream = getattr(sys, stream_name) tag = f"python.{stream_name}" self.assertEqual(f"", repr(stream)) - self.assertTrue(stream.writable()) - self.assertFalse(stream.readable()) + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) self.assertEqual("UTF-8", stream.encoding) - self.assertTrue(stream.line_buffering) - self.assertFalse(stream.write_through) + self.assertIs(stream.line_buffering, True) + self.assertIs(stream.write_through, False) # stderr is backslashreplace by default; stdout is configured # that way by libregrtest.main. @@ -147,6 +171,13 @@ def write(s, lines=None, *, write_len=None): write("f\n\ng", ["exxf", ""]) write("\n", ["g"]) + # Since this is a line-based logging system, line buffering + # cannot be turned off, i.e. a newline always causes a flush. + stream.reconfigure(line_buffering=False) + self.assertIs(stream.line_buffering, True) + + # However, buffering can be turned off completely if you want a + # flush after every write. with self.unbuffered(stream): write("\nx", ["", "x"]) write("\na\n", ["", "a"]) @@ -209,30 +240,30 @@ def __str__(self): # (MAX_BYTES_PER_WRITE). # # ASCII (1 byte per character) - write(("foobar" * 700) + "\n", - [("foobar" * 666) + "foob", # 4000 bytes - "ar" + ("foobar" * 33)]) # 200 bytes + write(("foobar" * 700) + "\n", # 4200 bytes in + [("foobar" * 666) + "foob", # 4000 bytes out + "ar" + ("foobar" * 33)]) # 200 bytes out # "Full-width" digits 0-9 (3 bytes per character) s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" - write((s * 150) + "\n", - [s * 100, # 3000 bytes - s * 50]) # 1500 bytes + write((s * 150) + "\n", # 4500 bytes in + [s * 100, # 3000 bytes out + s * 50]) # 1500 bytes out s = "0123456789" - write(s * 200, []) - write(s * 150, []) - write(s * 51, [s * 350]) # 3500 bytes - write("\n", [s * 51]) # 510 bytes + write(s * 200, []) # 2000 bytes in + write(s * 150, []) # 1500 bytes in + write(s * 51, [s * 350]) # 510 bytes in, 3500 bytes out + write("\n", [s * 51]) # 0 bytes in, 510 bytes out def test_bytes(self): for stream_name, level in [("stdout", "I"), ("stderr", "W")]: - with self.subTest(stream=stream_name): + with self.stream_context(stream_name, level): stream = getattr(sys, stream_name).buffer tag = f"python.{stream_name}" self.assertEqual(f"", repr(stream)) - self.assertTrue(stream.writable()) - self.assertFalse(stream.readable()) + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) def write(b, lines=None, *, write_len=None): if write_len is None: @@ -330,3 +361,64 @@ def write(b, lines=None, *, write_len=None): fr"{type(obj).__name__}" ): stream.write(obj) + + def test_rate_limit(self): + # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 + PER_MESSAGE_OVERHEAD = 28 + + # https://developer.android.com/ndk/reference/group/logging + ANDROID_LOG_DEBUG = 3 + + # To avoid flooding the test script output, use a different tag rather + # than stdout or stderr. + tag = "python.rate_limit" + stream = TextLogStream(ANDROID_LOG_DEBUG, tag) + + # Make a test message which consumes 1 KB of the logcat buffer. + message = "Line {:03d} " + message += "." * ( + 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0)) + ) + "\n" + + # See _android_support.py. The default values of these parameters work + # well across a wide range of devices, but we'll use smaller values to + # ensure a quick and reliable test that doesn't flood the log too much. + MAX_KB_PER_SECOND = 100 + BUCKET_KB = 10 + with ( + patch("_android_support.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), + patch("_android_support.BUCKET_SIZE", BUCKET_KB * 1024), + ): + # Make sure the token bucket is full. + sleep(BUCKET_KB / MAX_KB_PER_SECOND) + line_num = 0 + + # Write BUCKET_KB messages, and return the rate at which they were + # accepted in KB per second. + def write_bucketful(): + nonlocal line_num + start = time() + max_line_num = line_num + BUCKET_KB + while line_num < max_line_num: + stream.write(message.format(line_num)) + line_num += 1 + return BUCKET_KB / (time() - start) + + # The first bucketful should be written with minimal delay. The + # factor of 2 here is not arbitrary: it verifies that the system can + # write fast enough to empty the bucket within two bucketfuls, which + # the next part of the test depends on. + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) + + # Write another bucketful to empty the token bucket completely. + write_bucketful() + + # The next bucketful should be written at the rate limit. + self.assertAlmostEqual( + write_bucketful(), MAX_KB_PER_SECOND, + delta=MAX_KB_PER_SECOND * 0.1 + ) + + # Once the token bucket refills, we should go back to full speed. + sleep(BUCKET_KB / MAX_KB_PER_SECOND) + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-08-05-19-04-06.gh-issue-116622.3LWUzE.rst b/Misc/NEWS.d/next/Core and Builtins/2024-08-05-19-04-06.gh-issue-116622.3LWUzE.rst new file mode 100644 index 00000000000000..9320928477af2c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-08-05-19-04-06.gh-issue-116622.3LWUzE.rst @@ -0,0 +1 @@ +Fix Android stdout and stderr messages being truncated or lost. From 94a4bd79a7ab7b0ff5f216782d6fdaff6ed348fc Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Tue, 6 Aug 2024 08:57:36 +0300 Subject: [PATCH 3/6] gh-122704: Fix reference leak in Modules/_pickle.c (GH-122705) --- Modules/_pickle.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 5d9ee8cb6c679d..dc0ef0a184d205 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1962,9 +1962,11 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * PyErr_Format(st->PicklingError, "Can't pickle %R: import of module %R failed", global, module_name); + Py_DECREF(module_name); return NULL; } if (check_dotted_path(module, global_name, dotted_path) < 0) { + Py_DECREF(module_name); Py_DECREF(module); return NULL; } @@ -1974,6 +1976,7 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * PyErr_Format(st->PicklingError, "Can't pickle %R: attribute lookup %S on %S failed", global, global_name, module_name); + Py_DECREF(module_name); return NULL; } if (actual != global) { @@ -1981,6 +1984,7 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * PyErr_Format(st->PicklingError, "Can't pickle %R: it's not the same object as %S.%S", global, module_name, global_name); + Py_DECREF(module_name); return NULL; } Py_DECREF(actual); From e74680b7186e6823ea37cf7ab326d3d6bfa6f59a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 6 Aug 2024 08:59:44 +0300 Subject: [PATCH 4/6] gh-122595: Add more error checks in the compiler (GH-122596) --- Python/compile.c | 78 +++++++++++++++++++++--------- Python/symtable.c | 119 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 145 insertions(+), 52 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 87b2c2705474a4..9695a99d201144 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -505,21 +505,35 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) deterministic, then the generated bytecode is not deterministic. */ sorted_keys = PyDict_Keys(src); - if (sorted_keys == NULL) + if (sorted_keys == NULL) { + Py_DECREF(dest); return NULL; + } if (PyList_Sort(sorted_keys) != 0) { Py_DECREF(sorted_keys); + Py_DECREF(dest); return NULL; } num_keys = PyList_GET_SIZE(sorted_keys); for (key_i = 0; key_i < num_keys; key_i++) { - /* XXX this should probably be a macro in symtable.h */ - long vi; k = PyList_GET_ITEM(sorted_keys, key_i); v = PyDict_GetItemWithError(src, k); - assert(v && PyLong_Check(v)); - vi = PyLong_AS_LONG(v); + if (!v) { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, k); + } + Py_DECREF(sorted_keys); + Py_DECREF(dest); + return NULL; + } + long vi = PyLong_AsLong(v); + if (vi == -1 && PyErr_Occurred()) { + Py_DECREF(sorted_keys); + Py_DECREF(dest); + return NULL; + } + /* XXX this should probably be a macro in symtable.h */ scope = (vi >> SCOPE_OFFSET) & SCOPE_MASK; if (scope == scope_type || vi & flag) { @@ -631,6 +645,7 @@ compiler_set_qualname(struct compiler *c) scope = _PyST_GetScope(parent->u_ste, mangled); Py_DECREF(mangled); + RETURN_IF_ERROR(scope); assert(scope != GLOBAL_IMPLICIT); if (scope == GLOBAL_EXPLICIT) force_global = 1; @@ -1648,7 +1663,7 @@ dict_lookup_arg(PyObject *dict, PyObject *name) if (v == NULL) { return ERROR; } - return PyLong_AS_LONG(v); + return PyLong_AsLong(v); } static int @@ -1671,7 +1686,7 @@ compiler_lookup_arg(struct compiler *c, PyCodeObject *co, PyObject *name) else { arg = dict_lookup_arg(c->u->u_metadata.u_freevars, name); } - if (arg == -1) { + if (arg == -1 && !PyErr_Occurred()) { PyObject *freevars = _PyCode_GetFreevars(co); if (freevars == NULL) { PyErr_Clear(); @@ -4085,6 +4100,8 @@ compiler_nameop(struct compiler *c, location loc, case GLOBAL_EXPLICIT: optype = OP_GLOBAL; break; + case -1: + goto error; default: /* scope can be 0 */ break; @@ -4638,6 +4655,7 @@ is_import_originated(struct compiler *c, expr_ty e) } long flags = _PyST_GetSymbol(SYMTABLE(c)->st_top, e->v.Name.id); + RETURN_IF_ERROR(flags); return flags & DEF_IMPORT; } @@ -4657,10 +4675,12 @@ can_optimize_super_call(struct compiler *c, expr_ty attr) PyObject *super_name = e->v.Call.func->v.Name.id; // detect statically-visible shadowing of 'super' name int scope = _PyST_GetScope(SYMTABLE_ENTRY(c), super_name); + RETURN_IF_ERROR(scope); if (scope != GLOBAL_IMPLICIT) { return 0; } scope = _PyST_GetScope(SYMTABLE(c)->st_top, super_name); + RETURN_IF_ERROR(scope); if (scope != 0) { return 0; } @@ -4767,7 +4787,9 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) } /* Check that the base object is not something that is imported */ - if (is_import_originated(c, meth->v.Attribute.value)) { + int ret = is_import_originated(c, meth->v.Attribute.value); + RETURN_IF_ERROR(ret); + if (ret) { return 0; } @@ -4795,7 +4817,9 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) /* Alright, we can optimize the code. */ location loc = LOC(meth); - if (can_optimize_super_call(c, meth)) { + ret = can_optimize_super_call(c, meth); + RETURN_IF_ERROR(ret); + if (ret) { RETURN_IF_ERROR(load_args_for_super(c, meth->v.Attribute.value)); int opcode = asdl_seq_LEN(meth->v.Attribute.value->v.Call.args) ? LOAD_SUPER_METHOD : LOAD_ZERO_SUPER_METHOD; @@ -5367,8 +5391,10 @@ push_inlined_comprehension_state(struct compiler *c, location loc, PyObject *k, *v; Py_ssize_t pos = 0; while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { - assert(PyLong_Check(v)); - long symbol = PyLong_AS_LONG(v); + long symbol = PyLong_AsLong(v); + if (symbol == -1 && PyErr_Occurred()) { + return ERROR; + } long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; PyObject *outv = PyDict_GetItemWithError(SYMTABLE_ENTRY(c)->ste_symbols, k); if (outv == NULL) { @@ -5377,8 +5403,11 @@ push_inlined_comprehension_state(struct compiler *c, location loc, } outv = _PyLong_GetZero(); } - assert(PyLong_CheckExact(outv)); - long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; + long outsymbol = PyLong_AsLong(outv); + if (outsymbol == -1 && PyErr_Occurred()) { + return ERROR; + } + long outsc = (outsymbol >> SCOPE_OFFSET) & SCOPE_MASK; // If a name has different scope inside than outside the comprehension, // we need to temporarily handle it with the right scope while // compiling the comprehension. If it's free in the comprehension @@ -6064,14 +6093,18 @@ compiler_visit_expr(struct compiler *c, expr_ty e) return compiler_formatted_value(c, e); /* The following exprs can be assignment targets. */ case Attribute_kind: - if (e->v.Attribute.ctx == Load && can_optimize_super_call(c, e)) { - RETURN_IF_ERROR(load_args_for_super(c, e->v.Attribute.value)); - int opcode = asdl_seq_LEN(e->v.Attribute.value->v.Call.args) ? - LOAD_SUPER_ATTR : LOAD_ZERO_SUPER_ATTR; - ADDOP_NAME(c, loc, opcode, e->v.Attribute.attr, names); - loc = update_start_location_to_match_attr(c, loc, e); - ADDOP(c, loc, NOP); - return SUCCESS; + if (e->v.Attribute.ctx == Load) { + int ret = can_optimize_super_call(c, e); + RETURN_IF_ERROR(ret); + if (ret) { + RETURN_IF_ERROR(load_args_for_super(c, e->v.Attribute.value)); + int opcode = asdl_seq_LEN(e->v.Attribute.value->v.Call.args) ? + LOAD_SUPER_ATTR : LOAD_ZERO_SUPER_ATTR; + ADDOP_NAME(c, loc, opcode, e->v.Attribute.attr, names); + loc = update_start_location_to_match_attr(c, loc, e); + ADDOP(c, loc, NOP); + return SUCCESS; + } } RETURN_IF_ERROR(compiler_maybe_add_static_attribute_to_class(c, e)); VISIT(c, expr, e->v.Attribute.value); @@ -7300,7 +7333,8 @@ consts_dict_keys_inorder(PyObject *dict) if (consts == NULL) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { - i = PyLong_AS_LONG(v); + assert(PyLong_CheckExact(v)); + i = PyLong_AsLong(v); /* The keys of the dictionary can be tuples wrapping a constant. * (see dict_add_o and _PyCode_ConstantKey). In that case * the object we want is always second. */ diff --git a/Python/symtable.c b/Python/symtable.c index ef81a0799de3aa..4acf762f8fca39 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -526,17 +526,31 @@ _PySymtable_LookupOptional(struct symtable *st, void *key, long _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { - PyObject *v = PyDict_GetItemWithError(ste->ste_symbols, name); - if (!v) + PyObject *v; + if (PyDict_GetItemRef(ste->ste_symbols, name, &v) < 0) { + return -1; + } + if (!v) { return 0; - assert(PyLong_Check(v)); - return PyLong_AS_LONG(v); + } + long symbol = PyLong_AsLong(v); + Py_DECREF(v); + if (symbol < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, "invalid symbol"); + } + return -1; + } + return symbol; } int _PyST_GetScope(PySTEntryObject *ste, PyObject *name) { long symbol = _PyST_GetSymbol(ste, name); + if (symbol < 0) { + return -1; + } return (symbol >> SCOPE_OFFSET) & SCOPE_MASK; } @@ -715,11 +729,14 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, // global statement), we want to also treat it as a global in this scope. if (class_entry != NULL) { long class_flags = _PyST_GetSymbol(class_entry, name); + if (class_flags < 0) { + return 0; + } if (class_flags & DEF_GLOBAL) { SET_SCOPE(scopes, name, GLOBAL_EXPLICIT); return 1; } - else if (class_flags & DEF_BOUND && !(class_flags & DEF_NONLOCAL)) { + else if ((class_flags & DEF_BOUND) && !(class_flags & DEF_NONLOCAL)) { SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); return 1; } @@ -763,6 +780,9 @@ is_free_in_any_child(PySTEntryObject *entry, PyObject *key) PySTEntryObject *child_ste = (PySTEntryObject *)PyList_GET_ITEM( entry->ste_children, i); long scope = _PyST_GetScope(child_ste, key); + if (scope < 0) { + return -1; + } if (scope == FREE) { return 1; } @@ -781,7 +801,10 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter - long comp_flags = PyLong_AS_LONG(v); + long comp_flags = PyLong_AsLong(v); + if (comp_flags == -1 && PyErr_Occurred()) { + return 0; + } if (comp_flags & DEF_PARAM) { assert(_PyUnicode_EqualToASCIIString(k, ".0")); continue; @@ -822,11 +845,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, SET_SCOPE(scopes, k, scope); } else { - if (PyLong_AsLong(existing) & DEF_BOUND) { + long flags = PyLong_AsLong(existing); + if (flags == -1 && PyErr_Occurred()) { + return 0; + } + if ((flags & DEF_BOUND) && ste->ste_type != ClassBlock) { // free vars in comprehension that are locals in outer scope can // now simply be locals, unless they are free in comp children, // or if the outer scope is a class block - if (!is_free_in_any_child(comp, k) && ste->ste_type != ClassBlock) { + int ok = is_free_in_any_child(comp, k); + if (ok < 0) { + return 0; + } + if (!ok) { if (PySet_Discard(comp_free, k) < 0) { return 0; } @@ -861,9 +892,10 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) if (!v_cell) return 0; while (PyDict_Next(scopes, &pos, &name, &v)) { - long scope; - assert(PyLong_Check(v)); - scope = PyLong_AS_LONG(v); + long scope = PyLong_AsLong(v); + if (scope == -1 && PyErr_Occurred()) { + goto error; + } if (scope != LOCAL) continue; int contains = PySet_Contains(free, name); @@ -926,9 +958,10 @@ update_symbols(PyObject *symbols, PyObject *scopes, /* Update scope information for all symbols in this scope */ while (PyDict_Next(symbols, &pos, &name, &v)) { - long scope, flags; - assert(PyLong_Check(v)); - flags = PyLong_AS_LONG(v); + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + return 0; + } int contains = PySet_Contains(inlined_cells, name); if (contains < 0) { return 0; @@ -936,9 +969,18 @@ update_symbols(PyObject *symbols, PyObject *scopes, if (contains) { flags |= DEF_COMP_CELL; } - v_scope = PyDict_GetItemWithError(scopes, name); - assert(v_scope && PyLong_Check(v_scope)); - scope = PyLong_AS_LONG(v_scope); + if (PyDict_GetItemRef(scopes, name, &v_scope) < 0) { + return 0; + } + if (!v_scope) { + PyErr_SetObject(PyExc_KeyError, name); + return 0; + } + long scope = PyLong_AsLong(v_scope); + Py_DECREF(v_scope); + if (scope == -1 && PyErr_Occurred()) { + return 0; + } flags |= (scope << SCOPE_OFFSET); v_new = PyLong_FromLong(flags); if (!v_new) @@ -971,7 +1013,11 @@ update_symbols(PyObject *symbols, PyObject *scopes, or global in the class scope. */ if (classflag) { - long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS; + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + goto error; + } + flags |= DEF_FREE_CLASS; v_new = PyLong_FromLong(flags); if (!v_new) { goto error; @@ -1110,7 +1156,10 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { - long flags = PyLong_AS_LONG(v); + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + goto error; + } if (!analyze_name(ste, scopes, name, flags, bound, local, free, global, type_params, class_entry)) goto error; @@ -1395,9 +1444,12 @@ symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { PyObject *mangled = _Py_MaybeMangle(st->st_private, ste, name); if (!mangled) - return 0; + return -1; long ret = _PyST_GetSymbol(ste, mangled); Py_DECREF(mangled); + if (ret < 0) { + return -1; + } return ret; } @@ -1420,7 +1472,10 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s return 0; dict = ste->ste_symbols; if ((o = PyDict_GetItemWithError(dict, mangled))) { - val = PyLong_AS_LONG(o); + val = PyLong_AsLong(o); + if (val == -1 && PyErr_Occurred()) { + goto error; + } if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { /* Is it better to use 'mangled' or 'name' here? */ PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name); @@ -1466,16 +1521,20 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s if (flag & DEF_PARAM) { if (PyList_Append(ste->ste_varnames, mangled) < 0) goto error; - } else if (flag & DEF_GLOBAL) { + } else if (flag & DEF_GLOBAL) { /* XXX need to update DEF_GLOBAL for other flags too; perhaps only DEF_FREE_GLOBAL */ - val = flag; + val = 0; if ((o = PyDict_GetItemWithError(st->st_global, mangled))) { - val |= PyLong_AS_LONG(o); + val = PyLong_AsLong(o); + if (val == -1 && PyErr_Occurred()) { + goto error; + } } else if (PyErr_Occurred()) { goto error; } + val |= flag; o = PyLong_FromLong(val); if (o == NULL) goto error; @@ -2176,6 +2235,9 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) */ if (ste->ste_comprehension) { long target_in_scope = symtable_lookup_entry(st, ste, target_name); + if (target_in_scope < 0) { + return 0; + } if ((target_in_scope & DEF_COMP_ITER) && (target_in_scope & DEF_LOCAL)) { PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name); @@ -2188,6 +2250,9 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */ if (ste->ste_type == FunctionBlock) { long target_in_scope = symtable_lookup_entry(st, ste, target_name); + if (target_in_scope < 0) { + return 0; + } if (target_in_scope & DEF_GLOBAL) { if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) return 0; @@ -2601,9 +2666,6 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args) { Py_ssize_t i; - if (!args) - return -1; - for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); if (!symtable_add_def(st, arg->arg, DEF_PARAM, LOCATION(arg))) @@ -2650,9 +2712,6 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args) { Py_ssize_t i; - if (!args) - return -1; - for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); if (arg->annotation) { From b72c748d7fb4ecc0bc4626c7bc05fbc6c83f0ba8 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 5 Aug 2024 23:16:29 -0700 Subject: [PATCH 5/6] Fix syntax in generate_re_casefix.py (#122699) This was broken in gh-97963. --- Lib/re/_casefix.py | 2 +- Tools/build/generate_re_casefix.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/re/_casefix.py b/Lib/re/_casefix.py index 06507d08bee02b..fed2d84fc01473 100644 --- a/Lib/re/_casefix.py +++ b/Lib/re/_casefix.py @@ -1,4 +1,4 @@ -# Auto-generated by Tools/scripts/generate_re_casefix.py. +# Auto-generated by Tools/build/generate_re_casefix.py. # Maps the code of lowercased character to codes of different lowercased # characters which have the same uppercase. diff --git a/Tools/build/generate_re_casefix.py b/Tools/build/generate_re_casefix.py index b57ac07426c27c..6cebfbd025c58c 100755 --- a/Tools/build/generate_re_casefix.py +++ b/Tools/build/generate_re_casefix.py @@ -23,9 +23,9 @@ def update_file(file, content): # Maps the code of lowercased character to codes of different lowercased # characters which have the same uppercase. -_EXTRA_CASES = { +_EXTRA_CASES = {{ %s -} +}} """ def uname(i): From a8be8fc6c4682089be45a87bd5ee1f686040116c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 6 Aug 2024 08:40:39 +0100 Subject: [PATCH 6/6] GH-120024: Refactor code a bit so that escaping calls can be wrapped in spill code in code generator (GH-122693) --- Python/bytecodes.c | 50 ++++++++++++++++++++++++-------------- Python/executor_cases.c.h | 24 ++++++++++++------ Python/generated_cases.c.h | 46 +++++++++++++++++++++++------------ 3 files changed, 78 insertions(+), 42 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 996f997d0ca8de..a8527fe84b76f5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -287,7 +287,8 @@ dummy_func( /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyStackRef_GenCheck(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + if (err) { ERROR_NO_POP(); } } @@ -302,7 +303,8 @@ dummy_func( tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + if (err) { ERROR_NO_POP(); } } @@ -1069,11 +1071,12 @@ dummy_func( PyStackRef_AsPyObjectBorrow(v)); } if (retval_o == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (matches) { _PyEval_MonitorRaise(tstate, frame, this_instr); } - if (_PyGen_FetchStopIterationValue(&retval_o) == 0) { + int err = _PyGen_FetchStopIterationValue(&retval_o); + if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); } @@ -1210,7 +1213,8 @@ dummy_func( assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); DECREF_INPUTS(); none = PyStackRef_None; @@ -1425,7 +1429,8 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o; - if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) { + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + if (err < 0) { ERROR_NO_POP(); } if (v_o == NULL) { @@ -1596,7 +1601,8 @@ dummy_func( assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + if (err < 0) { ERROR_NO_POP(); } if (!value_o) { @@ -1676,7 +1682,8 @@ dummy_func( PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); @@ -1762,8 +1769,10 @@ dummy_func( PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (PyDict_Update(dict_o, update_o) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + int err = PyDict_Update(dict_o, update_o); + if (err < 0) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + if (matches) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); @@ -1779,7 +1788,8 @@ dummy_func( PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + int err = _PyDict_MergeEx(dict_o, update_o, 2); + if (err < 0) { _PyEval_FormatKwargsError(tstate, callable_o, update_o); DECREF_INPUTS(); ERROR_IF(true, error); @@ -1943,7 +1953,8 @@ dummy_func( if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr_o = NULL; - if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -2416,8 +2427,8 @@ dummy_func( inst(CHECK_EG_MATCH, (exc_value_st, match_type_st -- rest, match)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + if (err < 0) { DECREF_INPUTS(); ERROR_IF(true, error); } @@ -2704,7 +2715,8 @@ dummy_func( if (next_o == NULL) { next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { ERROR_NO_POP(); } _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -2729,7 +2741,8 @@ dummy_func( PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { ERROR_NO_POP(); } _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); @@ -2756,7 +2769,8 @@ dummy_func( } else { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { ERROR_NO_POP(); } _PyEval_MonitorRaise(tstate, frame, this_instr); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index cbee77d5cf67fc..7f520eb7abbe80 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1777,7 +1777,8 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + if (err < 0) { JUMP_TO_ERROR(); } if (!value_o) { @@ -1907,7 +1908,8 @@ PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); @@ -2031,8 +2033,10 @@ dict = stack_pointer[-2 - (oparg - 1)]; PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (PyDict_Update(dict_o, update_o) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + int err = PyDict_Update(dict_o, update_o); + if (err < 0) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + if (matches) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); @@ -2057,7 +2061,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + int err = _PyDict_MergeEx(dict_o, update_o, 2); + if (err < 0) { _PyEval_FormatKwargsError(tstate, callable_o, update_o); PyStackRef_CLOSE(update); if (true) JUMP_TO_ERROR(); @@ -2182,7 +2187,8 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr_o = NULL; - if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -2855,7 +2861,8 @@ exc_value_st = stack_pointer[-2]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (true) JUMP_TO_ERROR(); @@ -3101,7 +3108,8 @@ PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { JUMP_TO_ERROR(); } _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 879c40ab0cb6ba..31490960d3828a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2465,7 +2465,8 @@ exc_value_st = stack_pointer[-2]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (true) goto pop_2_error; @@ -2528,7 +2529,8 @@ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); PyStackRef_CLOSE(sub_iter_st); PyStackRef_CLOSE(last_sent_val_st); @@ -2982,7 +2984,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + int err = _PyDict_MergeEx(dict_o, update_o, 2); + if (err < 0) { _PyEval_FormatKwargsError(tstate, callable_o, update_o); PyStackRef_CLOSE(update); if (true) goto pop_1_error; @@ -3003,8 +3006,10 @@ dict = stack_pointer[-2 - (oparg - 1)]; PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (PyDict_Update(dict_o, update_o) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + int err = PyDict_Update(dict_o, update_o); + if (err < 0) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + if (matches) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); @@ -3208,7 +3213,8 @@ if (next_o == NULL) { next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { goto error; } _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -3786,7 +3792,8 @@ /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyStackRef_GenCheck(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + if (err) { goto error; } } @@ -3807,7 +3814,8 @@ receiver = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + if (err) { goto error; } } @@ -3834,7 +3842,8 @@ } else { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { goto error; } _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -4327,7 +4336,8 @@ PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); @@ -4379,7 +4389,8 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr_o = NULL; - if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -5074,7 +5085,8 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + if (err < 0) { goto error; } if (!value_o) { @@ -5100,7 +5112,8 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o; - if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) { + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + if (err < 0) { goto error; } if (v_o == NULL) { @@ -6080,11 +6093,12 @@ PyStackRef_AsPyObjectBorrow(v)); } if (retval_o == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (matches) { _PyEval_MonitorRaise(tstate, frame, this_instr); } - if (_PyGen_FetchStopIterationValue(&retval_o) == 0) { + int err = _PyGen_FetchStopIterationValue(&retval_o); + if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); }