diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index fb4a2218ddd8d8..63252a76abb69f 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -14,20 +14,6 @@ jobs: - template: ./prebuild-checks.yml -- job: Docs_PR - displayName: Docs PR - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) - - pool: - vmImage: ubuntu-22.04 - - steps: - - template: ./docs-steps.yml - parameters: - upload: true - - - job: macOS_CI_Tests displayName: macOS CI Tests dependsOn: Prebuild diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml deleted file mode 100644 index 647daff7a033a8..00000000000000 --- a/.azure-pipelines/docs-steps.yml +++ /dev/null @@ -1,47 +0,0 @@ -parameters: - latex: false - upload: false - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- task: UsePythonVersion@0 - displayName: 'Use Python 3.6 or later' - inputs: - versionSpec: '>=3.6' - -- script: python -m pip install -r requirements.txt - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Install build dependencies' - -- ${{ if ne(parameters.latex, 'true') }}: - - script: make check html PYTHON=python - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Build documentation' - -- ${{ if eq(parameters.latex, 'true') }}: - - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full - displayName: 'Install LaTeX' - - - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb' - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Build documentation' - -- ${{ if eq(parameters.upload, 'true') }}: - - task: PublishBuildArtifacts@1 - displayName: 'Publish docs' - - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/build' - ArtifactName: docs - publishLocation: Container - - - ${{ if eq(parameters.latex, 'true') }}: - - task: PublishBuildArtifacts@1 - displayName: 'Publish dist' - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/dist' - ArtifactName: docs_dist - publishLocation: Container diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index b822d58806b9a6..939c9b4249a3c2 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -14,18 +14,6 @@ jobs: - template: ./prebuild-checks.yml -- job: Docs_PR - displayName: Docs PR - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) - - pool: - vmImage: ubuntu-22.04 - - steps: - - template: ./docs-steps.yml - - - job: macOS_PR_Tests displayName: macOS PR Tests dependsOn: Prebuild diff --git a/.azure-pipelines/prebuild-checks.yml b/.azure-pipelines/prebuild-checks.yml index 30ff642d1267a1..2c6460d2386735 100644 --- a/.azure-pipelines/prebuild-checks.yml +++ b/.azure-pipelines/prebuild-checks.yml @@ -11,18 +11,6 @@ steps: displayName: Fetch comparison tree condition: and(succeeded(), variables['System.PullRequest.TargetBranch']) -- script: | - if ! git diff --name-only $(diffTarget) | grep -qE '(\.rst$|^Doc|^Misc)' - then - echo "No docs were updated: docs.run=false" - echo "##vso[task.setvariable variable=run;isOutput=true]false" - else - echo "Docs were updated: docs.run=true" - echo "##vso[task.setvariable variable=run;isOutput=true]true" - fi - displayName: Detect documentation changes - name: docs - - script: | if ! git diff --name-only $(diffTarget) | grep -qvE '(\.rst$|^Doc|^Misc)' then diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 32ba5355a5853f..95fd51b743cd33 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -164,4 +164,4 @@ Lib/ast.py @isidentical **/*pathlib* @barneygale # zipfile.Path -**/*zipfile/*_path.py @jaraco +**/*zipfile/_path/* @jaraco diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e1d4047a877739..197953b513f06f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -239,7 +239,7 @@ jobs: path: config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - name: Install Homebrew dependencies - run: brew install pkg-config openssl@1.1 xz gdbm tcl-tk + run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk - name: Configure CPython run: | GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ @@ -248,7 +248,7 @@ jobs: --config-cache \ --with-pydebug \ --prefix=/opt/python-dev \ - --with-openssl="$(brew --prefix openssl@1.1)" + --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython run: make -j4 - name: Display build info diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index ec900ce68a1dde..3211b526efc7a5 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -51,8 +51,6 @@ jobs: cache-dependency-path: 'Doc/requirements.txt' - name: 'Install build dependencies' run: make -C Doc/ venv - - name: 'Check documentation' - run: make -C Doc/ check - name: 'Build HTML documentation' run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html @@ -82,8 +80,6 @@ jobs: make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going" html 2>&1 # This build doesn't use problem matchers or check annotations - # It also does not run 'make check', as sphinx-lint is not installed into the - # environment. build_doc_oldest_supported_sphinx: name: 'Docs (Oldest Sphinx)' runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 808622f19a3dbf..d62c57c044728f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,5 +3,16 @@ repos: rev: v4.4.0 hooks: - id: check-yaml + - id: end-of-file-fixer + types: [python] + exclude: Lib/test/coding20731.py - id: trailing-whitespace types_or: [c, python, rst] + + - repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v0.6.7 + hooks: + - id: sphinx-lint + args: [--enable=default-role] + files: ^Doc/ + types: [rst] diff --git a/Doc/Makefile b/Doc/Makefile index c11ea6ce03e8a4..22691895068fea 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -216,11 +216,9 @@ dist: rm dist/python-$(DISTVERSION)-docs-texinfo.tar .PHONY: check -check: - # Check the docs and NEWS files with sphinx-lint. - # Ignore the tools and venv dirs and check that the default role is not used. - $(SPHINXLINT) -i tools -i $(VENVDIR) --enable default-role - $(SPHINXLINT) --enable default-role ../Misc/NEWS.d/next/ +check: venv + $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit + $(VENVDIR)/bin/python3 -m pre_commit run --all-files .PHONY: serve serve: diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst index c197d447e9618c..b2d8f2124fc203 100644 --- a/Doc/c-api/bool.rst +++ b/Doc/c-api/bool.rst @@ -6,7 +6,7 @@ Boolean Objects --------------- Booleans in Python are implemented as a subclass of integers. There are only -two booleans, :const:`Py_False` and :const:`Py_True`. As such, the normal +two booleans, :c:data:`Py_False` and :c:data:`Py_True`. As such, the normal creation and deletion functions don't apply to booleans. The following macros are available, however. @@ -19,29 +19,32 @@ are available, however. .. c:var:: PyObject* Py_False - The Python ``False`` object. This object has no methods. It needs to be - treated just like any other object with respect to reference counts. + The Python ``False`` object. This object has no methods and is + `immortal `_. + +.. versionchanged:: 3.12 + :c:data:`Py_False` is immortal. .. c:var:: PyObject* Py_True - The Python ``True`` object. This object has no methods. It needs to be treated - just like any other object with respect to reference counts. + The Python ``True`` object. This object has no methods and is + `immortal `_. + +.. versionchanged:: 3.12 + :c:data:`Py_True` is immortal. .. c:macro:: Py_RETURN_FALSE - Return :const:`Py_False` from a function, properly incrementing its reference - count. + Return :c:data:`Py_False` from a function. .. c:macro:: Py_RETURN_TRUE - Return :const:`Py_True` from a function, properly incrementing its reference - count. + Return :c:data:`Py_True` from a function. .. c:function:: PyObject* PyBool_FromLong(long v) - Return a new reference to :const:`Py_True` or :const:`Py_False` depending on the - truth value of *v*. + Return :c:data:`Py_True` or :c:data:`Py_False`, depending on the truth value of *v*. diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 344da903da4c1a..cb8b270fcbab6e 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -127,12 +127,12 @@ Complex Numbers as Python Objects Return the :c:type:`Py_complex` value of the complex number *op*. - If *op* is not a Python complex number object but has a :meth:`__complex__` + If *op* is not a Python complex number object but has a :meth:`~object.__complex__` method, this method will first be called to convert *op* to a Python complex - number object. If ``__complex__()`` is not defined then it falls back to - :meth:`__float__`. If ``__float__()`` is not defined then it falls back - to :meth:`__index__`. Upon failure, this method returns ``-1.0`` as a real + number object. If :meth:`!__complex__` is not defined then it falls back to + :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + to :meth:`~object.__index__`. Upon failure, this method returns ``-1.0`` as a real value. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 0ca8ad624b2034..bd0c36a217e2ce 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -98,9 +98,11 @@ Dictionary Objects Return the object from dictionary *p* which has a key *key*. Return ``NULL`` if the key *key* is not present, but *without* setting an exception. - Note that exceptions which occur while calling :meth:`__hash__` and - :meth:`__eq__` methods will get suppressed. - To get error reporting use :c:func:`PyDict_GetItemWithError()` instead. + .. note:: + + Exceptions that occur while this calls :meth:`~object.__hash__` and + :meth:`~object.__eq__` methods are silently ignored. + Prefer the :c:func:`PyDict_GetItemWithError` function instead. .. versionchanged:: 3.10 Calling this API without :term:`GIL` held had been allowed for historical @@ -120,10 +122,13 @@ Dictionary Objects This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a :c:expr:`const char*`, rather than a :c:expr:`PyObject*`. - Note that exceptions which occur while calling :meth:`__hash__` and - :meth:`__eq__` methods and creating a temporary string object - will get suppressed. - To get error reporting use :c:func:`PyDict_GetItemWithError()` instead. + .. note:: + + Exceptions that occur while this calls :meth:`~object.__hash__` and + :meth:`~object.__eq__` methods or while creating the temporary :class:`str` + object are silently ignored. + Prefer using the :c:func:`PyDict_GetItemWithError` function with your own + :c:func:`PyUnicode_FromString` *key* instead. .. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *defaultobj) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 05b2d100d575cb..fd0be1108c6300 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -3,7 +3,7 @@ .. _floatobjects: Floating Point Objects ----------------------- +====================== .. index:: pair: object; floating point @@ -45,14 +45,14 @@ Floating Point Objects .. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) Return a C :c:expr:`double` representation of the contents of *pyfloat*. If - *pyfloat* is not a Python floating point object but has a :meth:`__float__` + *pyfloat* is not a Python floating point object but has a :meth:`~object.__float__` method, this method will first be called to convert *pyfloat* into a float. - If ``__float__()`` is not defined then it falls back to :meth:`__index__`. + If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. This method returns ``-1.0`` upon failure, so one should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. c:function:: double PyFloat_AS_DOUBLE(PyObject *pyfloat) @@ -79,7 +79,7 @@ Floating Point Objects Pack and Unpack functions -========================= +------------------------- The pack and unpack functions provide an efficient platform-independent way to store floating-point values as byte strings. The Pack routines produce a bytes @@ -104,7 +104,7 @@ happens in such cases is partly accidental (alas). .. versionadded:: 3.11 Pack functions --------------- +^^^^^^^^^^^^^^ The pack routines write 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if you want the bytes string in little-endian @@ -135,7 +135,7 @@ There are two problems on non-IEEE platforms: Unpack functions ----------------- +^^^^^^^^^^^^^^^^ The unpack routines read 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if the bytes string is in little-endian format diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 9f7addfbbf3cb4..1accee2767a485 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -134,7 +134,7 @@ See also :ref:`Reflection `. Internal Frames ---------------- +^^^^^^^^^^^^^^^ Unless using :pep:`523`, you will not need this. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 5c1d026a330ae7..fe379ffe912391 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -121,7 +121,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. single: OverflowError (built-in exception) Return a C :c:expr:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method + instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. Raise :exc:`OverflowError` if the value of *obj* is out of range for a @@ -130,16 +130,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow) Return a C :c:expr:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` + instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. If the value of *obj* is greater than :const:`LONG_MAX` or less than @@ -150,10 +150,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: long long PyLong_AsLongLong(PyObject *obj) @@ -162,7 +162,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. single: OverflowError (built-in exception) Return a C :c:expr:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method + instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. Raise :exc:`OverflowError` if the value of *obj* is out of range for a @@ -171,16 +171,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: long long PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow) Return a C :c:expr:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method + instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. If the value of *obj* is greater than :const:`LLONG_MAX` or less than @@ -193,10 +193,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.2 .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong) @@ -267,7 +267,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: unsigned long PyLong_AsUnsignedLongMask(PyObject *obj) Return a C :c:expr:`unsigned long` representation of *obj*. If *obj* is not - an instance of :c:type:`PyLongObject`, first call its :meth:`__index__` + an instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. If the value of *obj* is out of range for an :c:expr:`unsigned long`, @@ -277,17 +277,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. disambiguate. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj) Return a C :c:expr:`unsigned long long` representation of *obj*. If *obj* is not an instance of :c:type:`PyLongObject`, first call its - :meth:`__index__` method (if present) to convert it to a + :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. If the value of *obj* is out of range for an :c:expr:`unsigned long long`, @@ -297,10 +297,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. to disambiguate. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: double PyLong_AsDouble(PyObject *pylong) diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst index b84a16a28ead56..1a497652ac5655 100644 --- a/Doc/c-api/none.rst +++ b/Doc/c-api/none.rst @@ -15,12 +15,12 @@ same reason. .. c:var:: PyObject* Py_None - The Python ``None`` object, denoting lack of value. This object has no methods. - It needs to be treated just like any other object with respect to reference - counts. + The Python ``None`` object, denoting lack of value. This object has no methods + and is `immortal `_. +.. versionchanged:: 3.12 + :c:data:`Py_None` is immortal. .. c:macro:: Py_RETURN_NONE - Properly handle returning :c:data:`Py_None` from within a C function (that is, - increment the reference count of ``None`` and return it.) + Return :c:data:`Py_None` from a function. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a25ff244c9f07c..22e7458013fb3d 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -33,9 +33,11 @@ Object Protocol is equivalent to the Python expression ``hasattr(o, attr_name)``. This function always succeeds. - Note that exceptions which occur while calling :meth:`__getattr__` and - :meth:`__getattribute__` methods will get suppressed. - To get error reporting use :c:func:`PyObject_GetAttr()` instead. + .. note:: + + Exceptions that occur when this calls :meth:`~object.__getattr__` and + :meth:`~object.__getattribute__` methods are silently ignored. + For proper error handling, use :c:func:`PyObject_GetAttr` instead. .. c:function:: int PyObject_HasAttrString(PyObject *o, const char *attr_name) @@ -44,10 +46,12 @@ Object Protocol is equivalent to the Python expression ``hasattr(o, attr_name)``. This function always succeeds. - Note that exceptions which occur while calling :meth:`__getattr__` and - :meth:`__getattribute__` methods and creating a temporary string object - will get suppressed. - To get error reporting use :c:func:`PyObject_GetAttrString()` instead. + .. note:: + + Exceptions that occur when this calls :meth:`~object.__getattr__` and + :meth:`~object.__getattribute__` methods or while creating the temporary :class:`str` + object are silently ignored. + For proper error handling, use :c:func:`PyObject_GetAttrString` instead. .. c:function:: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name) diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 8271d9acfb645e..c54a659cf2ffd8 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -113,11 +113,14 @@ Slice Objects Ellipsis Object ---------------- +^^^^^^^^^^^^^^^ .. c:var:: PyObject *Py_Ellipsis - The Python ``Ellipsis`` object. This object has no methods. It needs to be - treated just like any other object with respect to reference counts. Like - :c:data:`Py_None` it is a singleton object. + The Python ``Ellipsis`` object. This object has no methods. Like + :c:data:`Py_None`, it is an `immortal `_. + singleton object. + + .. versionchanged:: 3.12 + :c:data:`Py_Ellipsis` is immortal. diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index bf261b9814456e..a5f333e2a31e03 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -50,6 +50,23 @@ Type Objects The return type is now ``unsigned long`` rather than ``long``. +.. c:function:: PyObject* PyType_GetDict(PyTypeObject* type) + + Return the type object's internal namespace, which is otherwise only + exposed via a read-only proxy (``cls.__dict__``). This is a + replacement for accessing :c:member:`~PyTypeObject.tp_dict` directly. + The returned dictionary must be treated as read-only. + + This function is meant for specific embedding and language-binding cases, + where direct access to the dict is necessary and indirect access + (e.g. via the proxy or :c:func:`PyObject_GetAttr`) isn't adequate. + + Extension modules should continue to use ``tp_dict``, + directly or indirectly, when setting up their own types. + + .. versionadded:: 3.12 + + .. c:function:: void PyType_Modified(PyTypeObject *type) Invalidate the internal lookup cache for the type and all of its @@ -258,7 +275,7 @@ The following functions and structs are used to create (or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below). Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not - supported. + supported, except if ``tp_new`` is ``NULL``. (For backwards compatibility, other ``PyType_From*`` functions allow such metaclasses. They ignore ``tp_new``, which may result in incomplete initialization. This is deprecated and in Python 3.14+ such metaclasses will diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index c6e783acdf0654..7249cfe79c32e9 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1143,14 +1143,14 @@ and :c:type:`PyType_Type` effectively act as defaults.) :const:`Py_TPFLAGS_IMMUTABLETYPE` flag set. For extension types, it is inherited whenever :c:member:`~PyTypeObject.tp_descr_get` is inherited. - .. data:: Py_TPFLAGS_MANAGED_DICT + .. data:: Py_TPFLAGS_MANAGED_DICT - This bit indicates that instances of the class have a ``__dict__`` - attribute, and that the space for the dictionary is managed by the VM. + This bit indicates that instances of the class have a ``__dict__`` + attribute, and that the space for the dictionary is managed by the VM. - If this flag is set, :const:`Py_TPFLAGS_HAVE_GC` should also be set. + If this flag is set, :const:`Py_TPFLAGS_HAVE_GC` should also be set. - .. versionadded:: 3.12 + .. versionadded:: 3.12 **Inheritance:** @@ -1158,12 +1158,12 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_dictoffset` field is set in a superclass. - .. data:: Py_TPFLAGS_MANAGED_WEAKREF + .. data:: Py_TPFLAGS_MANAGED_WEAKREF - This bit indicates that instances of the class should be weakly - referenceable. + This bit indicates that instances of the class should be weakly + referenceable. - .. versionadded:: 3.12 + .. versionadded:: 3.12 **Inheritance:** @@ -1717,7 +1717,19 @@ and :c:type:`PyType_Type` effectively act as defaults.) called; it may also be initialized to a dictionary containing initial attributes for the type. Once :c:func:`PyType_Ready` has initialized the type, extra attributes for the type may be added to this dictionary only if they don't - correspond to overloaded operations (like :meth:`__add__`). + correspond to overloaded operations (like :meth:`__add__`). Once + initialization for the type has finished, this field should be + treated as read-only. + + Some types may not store their dictionary in this slot. + Use :c:func:`PyType_GetDict` to retreive the dictionary for an arbitrary + type. + + .. versionchanged:: 3.12 + + Internals detail: For static builtin types, this is always ``NULL``. + Instead, the dict for such types is stored on ``PyInterpreterState``. + Use :c:func:`PyType_GetDict` to get the dict for an arbitrary type. **Inheritance:** diff --git a/Doc/constraints.txt b/Doc/constraints.txt index 66c748eb092d83..54888eaab242ee 100644 --- a/Doc/constraints.txt +++ b/Doc/constraints.txt @@ -23,7 +23,3 @@ sphinxcontrib-serializinghtml<1.2 # Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above) MarkupSafe<2.2 - -# Direct dependencies of sphinx-lint -polib<1.3 -regex<2024 diff --git a/Doc/data/python3.12.abi b/Doc/data/python3.12.abi index 0615923c38b667..4cd130f86e2f7e 100644 --- a/Doc/data/python3.12.abi +++ b/Doc/data/python3.12.abi @@ -1,26461 +1,26470 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 6852a385f0c63c..7f8f8ddaaaccd6 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -168,7 +168,7 @@ representation of the instance for which it is called. Here is a simple example:: static PyObject * - newdatatype_repr(newdatatypeobject * obj) + newdatatype_repr(newdatatypeobject *obj) { return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:%d}}", obj->obj_UnderlyingDatatypePtr->size); @@ -188,7 +188,7 @@ used instead. Here is a simple example:: static PyObject * - newdatatype_str(newdatatypeobject * obj) + newdatatype_str(newdatatypeobject *obj) { return PyUnicode_FromFormat("Stringified_newdatatype{{size:%d}}", obj->obj_UnderlyingDatatypePtr->size); @@ -338,7 +338,7 @@ Here is an example:: PyErr_Format(PyExc_AttributeError, "'%.100s' object has no attribute '%.400s'", - tp->tp_name, name); + Py_TYPE(obj)->tp_name, name); return NULL; } @@ -379,7 +379,7 @@ Here is a sample implementation, for a datatype that is considered equal if the size of an internal pointer is equal:: static PyObject * - newdatatype_richcmp(PyObject *obj1, PyObject *obj2, int op) + newdatatype_richcmp(newdatatypeobject *obj1, newdatatypeobject *obj2, int op) { PyObject *result; int c, size1, size2; @@ -478,7 +478,7 @@ This function takes three arguments: Here is a toy ``tp_call`` implementation:: static PyObject * - newdatatype_call(newdatatypeobject *self, PyObject *args, PyObject *kwds) + newdatatype_call(newdatatypeobject *obj, PyObject *args, PyObject *kwds) { PyObject *result; const char *arg1; diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index a9b2622e02ef3b..8727332594bda6 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -135,7 +135,7 @@ Python versions are numbered "A.B.C" or "A.B": See :pep:`6` for more information about bugfix releases. -Not all releases are bugfix releases. In the run-up to a new major release, a +Not all releases are bugfix releases. In the run-up to a new feature release, a series of development releases are made, denoted as alpha, beta, or release candidate. Alphas are early releases in which interfaces aren't yet finalized; it's not unexpected to see an interface change between two alpha releases. @@ -297,9 +297,9 @@ How stable is Python? Very stable. New, stable releases have been coming out roughly every 6 to 18 months since 1991, and this seems likely to continue. As of version 3.9, -Python will have a major new release every 12 months (:pep:`602`). +Python will have a new feature release every 12 months (:pep:`602`). -The developers issue "bugfix" releases of older versions, so the stability of +The developers issue bugfix releases of older versions, so the stability of existing releases gradually improves. Bugfix releases, indicated by a third component of the version number (e.g. 3.5.3, 3.6.2), are managed for stability; only fixes for known problems are included in a bugfix release, and it's diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 53e8cdcae1cd66..5c0f0f15217a07 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -92,8 +92,8 @@ Glossary asynchronous context manager An object which controls the environment seen in an - :keyword:`async with` statement by defining :meth:`__aenter__` and - :meth:`__aexit__` methods. Introduced by :pep:`492`. + :keyword:`async with` statement by defining :meth:`~object.__aenter__` and + :meth:`~object.__aexit__` methods. Introduced by :pep:`492`. asynchronous generator A function which returns an :term:`asynchronous generator iterator`. It @@ -113,26 +113,26 @@ Glossary An object created by a :term:`asynchronous generator` function. This is an :term:`asynchronous iterator` which when called using the - :meth:`__anext__` method returns an awaitable object which will execute + :meth:`~object.__anext__` method returns an awaitable object which will execute the body of the asynchronous generator function until the next :keyword:`yield` expression. Each :keyword:`yield` temporarily suspends processing, remembering the location execution state (including local variables and pending try-statements). When the *asynchronous generator iterator* effectively - resumes with another awaitable returned by :meth:`__anext__`, it + resumes with another awaitable returned by :meth:`~object.__anext__`, it picks up where it left off. See :pep:`492` and :pep:`525`. asynchronous iterable An object, that can be used in an :keyword:`async for` statement. Must return an :term:`asynchronous iterator` from its - :meth:`__aiter__` method. Introduced by :pep:`492`. + :meth:`~object.__aiter__` method. Introduced by :pep:`492`. asynchronous iterator - An object that implements the :meth:`__aiter__` and :meth:`__anext__` - methods. ``__anext__`` must return an :term:`awaitable` object. + An object that implements the :meth:`~object.__aiter__` and :meth:`~object.__anext__` + methods. :meth:`~object.__anext__` must return an :term:`awaitable` object. :keyword:`async for` resolves the awaitables returned by an asynchronous - iterator's :meth:`__anext__` method until it raises a + iterator's :meth:`~object.__anext__` method until it raises a :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. attribute @@ -149,7 +149,7 @@ Glossary awaitable An object that can be used in an :keyword:`await` expression. Can be - a :term:`coroutine` or an object with an :meth:`__await__` method. + a :term:`coroutine` or an object with an :meth:`~object.__await__` method. See also :pep:`492`. BDFL diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 4620b4617e3450..0f99cb64994ab2 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -27,7 +27,8 @@ Argument Clinic How-To version of Argument Clinic that ships with the next version of CPython *could* be totally incompatible and break all your code. -The Goals Of Argument Clinic + +The goals of Argument Clinic ============================ Argument Clinic's primary goal @@ -78,7 +79,7 @@ and it should be able to do many interesting and smart things with all the information you give it. -Basic Concepts And Usage +Basic concepts and usage ======================== Argument Clinic ships with CPython; you'll find it in ``Tools/clinic/clinic.py``. @@ -141,7 +142,7 @@ For the sake of clarity, here's the terminology we'll use with Argument Clinic: a block.) -Converting Your First Function +Converting your first function ============================== The best way to get a sense of how Argument Clinic works is to @@ -558,7 +559,8 @@ Let's dive in! Congratulations, you've ported your first function to work with Argument Clinic! -Advanced Topics + +Advanced topics =============== Now that you've had some experience working with Argument Clinic, it's time @@ -636,7 +638,8 @@ after the last argument). Currently the generated code will use :c:func:`PyArg_ParseTuple`, but this will change soon. -Optional Groups + +Optional groups --------------- Some legacy functions have a tricky approach to parsing their arguments: @@ -899,6 +902,7 @@ available. For each converter it'll show you all the parameters it accepts, along with the default value for each parameter. Just run ``Tools/clinic/clinic.py --converters`` to see the full list. + Py_buffer --------- @@ -908,7 +912,6 @@ you *must* not call :c:func:`PyBuffer_Release` on the provided buffer. Argument Clinic generates code that does it for you (in the parsing function). - Advanced converters ------------------- @@ -975,6 +978,7 @@ value called ``NULL`` for just this reason: from Python's perspective it behaves like a default value of ``None``, but the C variable is initialized with ``NULL``. + Expressions specified as default values --------------------------------------- @@ -1032,7 +1036,6 @@ you're not permitted to use: * Tuple/list/set/dict literals. - Using a return converter ------------------------ @@ -1146,6 +1149,7 @@ then modifying it. Cloning is an all-or nothing proposition. Also, the function you are cloning from must have been previously defined in the current file. + Calling Python code ------------------- @@ -1380,6 +1384,7 @@ handle initialization and cleanup. You can see more examples of custom converters in the CPython source tree; grep the C files for the string ``CConverter``. + Writing a custom return converter --------------------------------- @@ -1394,8 +1399,9 @@ write your own return converter, please read ``Tools/clinic/clinic.py``, specifically the implementation of ``CReturnConverter`` and all its subclasses. + METH_O and METH_NOARGS ----------------------------------------------- +---------------------- To convert a function using ``METH_O``, make sure the function's single argument is using the ``object`` converter, and mark the @@ -1415,8 +1421,9 @@ any arguments. You can still use a self converter, a return converter, and specify a ``type`` argument to the object converter for ``METH_O``. + tp_new and tp_init functions ----------------------------------------------- +---------------------------- You can convert ``tp_new`` and ``tp_init`` functions. Just name them ``__new__`` or ``__init__`` as appropriate. Notes: @@ -1437,6 +1444,7 @@ them ``__new__`` or ``__init__`` as appropriate. Notes: (If your function doesn't support keywords, the parsing function generated will throw an exception if it receives any.) + Changing and redirecting Clinic's output ---------------------------------------- @@ -1721,7 +1729,7 @@ the file was not modified by hand before it gets overwritten. The #ifdef trick ----------------------------------------------- +---------------- If you're converting a function that isn't available on all platforms, there's a trick you can use to make life a little easier. The existing @@ -1801,7 +1809,6 @@ Argument Clinic added to your file (it'll be at the very bottom), then move it above the ``PyMethodDef`` structure where that macro is used. - Using Argument Clinic in Python files ------------------------------------- diff --git a/Doc/install/index.rst b/Doc/install/index.rst index ab581d785ef7f0..beb34f0cf21b22 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -696,7 +696,7 @@ is supplied to suppress this behaviour. So you could simply edit import sys sys.path.append('/www/python/') -However, if you reinstall the same major version of Python (perhaps when +However, if you reinstall the same minor version of Python (perhaps when upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by the stock version. You'd have to remember that it was modified and save a copy before doing the installation. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index f3b0bf0c4f7779..530cf30643687f 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -146,6 +146,102 @@ Node classes Snakes `__ project and all its contributors. + +.. _ast-root-nodes: + +Root nodes +^^^^^^^^^^ + +.. class:: Module(body, type_ignores) + + A Python module, as with :ref:`file input `. + Node type generated by :func:`ast.parse` in the default ``"exec"`` *mode*. + + *body* is a :class:`list` of the module's :ref:`ast-statements`. + + *type_ignores* is a :class:`list` of the module's type ignore comments; + see :func:`ast.parse` for more details. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x = 1'), indent=4)) + Module( + body=[ + Assign( + targets=[ + Name(id='x', ctx=Store())], + value=Constant(value=1))], + type_ignores=[]) + + +.. class:: Expression(body) + + A single Python :ref:`expression input `. + Node type generated by :func:`ast.parse` when *mode* is ``"eval"``. + + *body* is a single node, + one of the :ref:`expression types `. + + .. doctest:: + + >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) + Expression( + body=Constant(value=123)) + + +.. class:: Interactive(body) + + A single :ref:`interactive input `, like in :ref:`tut-interac`. + Node type generated by :func:`ast.parse` when *mode* is ``"single"``. + + *body* is a :class:`list` of :ref:`statement nodes `. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x = 1; y = 2', mode='single'), indent=4)) + Interactive( + body=[ + Assign( + targets=[ + Name(id='x', ctx=Store())], + value=Constant(value=1)), + Assign( + targets=[ + Name(id='y', ctx=Store())], + value=Constant(value=2))]) + + +.. class:: FunctionType(argtypes, returns) + + A representation of an old-style type comments for functions, + as Python versions prior to 3.5 didn't support :pep:`484` annotations. + Node type generated by :func:`ast.parse` when *mode* is ``"func_type"``. + + Such type comments would look like this:: + + def sum_two_number(a, b): + # type: (int, int) -> int + return a + b + + *argtypes* is a :class:`list` of :ref:`expression nodes `. + + *returns* is a single :ref:`expression node `. + + .. doctest:: + + >>> print(ast.dump(ast.parse('(int, str) -> List[int]', mode='func_type'), indent=4)) + FunctionType( + argtypes=[ + Name(id='int', ctx=Load()), + Name(id='str', ctx=Load())], + returns=Subscript( + value=Name(id='List', ctx=Load()), + slice=Name(id='int', ctx=Load()), + ctx=Load())) + + .. versionadded:: 3.8 + + Literals ^^^^^^^^ @@ -344,6 +440,8 @@ Variables type_ignores=[]) +.. _ast-expressions: + Expressions ^^^^^^^^^^^ @@ -735,6 +833,9 @@ Comprehensions ifs=[], is_async=1)])) + +.. _ast-statements: + Statements ^^^^^^^^^^ diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 8d0022cc66daac..38f2e2f510c176 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -895,6 +895,9 @@ TLS Upgrade object only because the coder caches *protocol*-side data and sporadically exchanges extra TLS session packets with *transport*. + In some situations (e.g. when the passed transport is already closing) this + may return ``None``. + Parameters: * *transport* and *protocol* instances that methods like diff --git a/Doc/library/asyncio-exceptions.rst b/Doc/library/asyncio-exceptions.rst index 9250f01b8a0895..7ad9103ca3fdfc 100644 --- a/Doc/library/asyncio-exceptions.rst +++ b/Doc/library/asyncio-exceptions.rst @@ -31,7 +31,7 @@ Exceptions .. versionchanged:: 3.8 - :exc:`CancelledError` is now a subclass of :class:`BaseException`. + :exc:`CancelledError` is now a subclass of :class:`BaseException` rather than :class:`Exception`. .. exception:: InvalidStateError diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index fe8d028150403d..d07fbae287e535 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -651,16 +651,16 @@ Timeouts If ``long_running_task`` takes more than 10 seconds to complete, the context manager will cancel the current task and handle the resulting :exc:`asyncio.CancelledError` internally, transforming it - into an :exc:`asyncio.TimeoutError` which can be caught and handled. + into a :exc:`TimeoutError` which can be caught and handled. .. note:: The :func:`asyncio.timeout` context manager is what transforms - the :exc:`asyncio.CancelledError` into an :exc:`asyncio.TimeoutError`, - which means the :exc:`asyncio.TimeoutError` can only be caught + the :exc:`asyncio.CancelledError` into a :exc:`TimeoutError`, + which means the :exc:`TimeoutError` can only be caught *outside* of the context manager. - Example of catching :exc:`asyncio.TimeoutError`:: + Example of catching :exc:`TimeoutError`:: async def main(): try: diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index b17d58e1cc0ce1..fdac51d9603ceb 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -9,7 +9,7 @@ This module provides access to mathematical functions for complex numbers. The functions in this module accept integers, floating-point numbers or complex numbers as arguments. They will also accept any Python object that has either a -:meth:`__complex__` or a :meth:`__float__` method: these methods are used to +:meth:`~object.__complex__` or a :meth:`~object.__float__` method: these methods are used to convert the object to a complex or floating-point number, respectively, and the function is then applied to the result of the conversion. diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index 740084b40c5ac9..51f91463f5ff93 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -20,6 +20,7 @@ The :mod:`crypt` module is deprecated (see :pep:`PEP 594 <594#crypt>` for details and alternatives). The :mod:`hashlib` module is a potential replacement for certain use cases. + The `passlib `_ package can replace all use cases of this module. -------------- diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index a5b20149921042..535a60ccca8d07 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -738,7 +738,7 @@ for ``x`` when creating a class instance will share the same copy of ``x``. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`dataclass` decorator will raise a :exc:`TypeError` if it +:func:`dataclass` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 0b4a4973cb4da0..018ec1abf36406 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -743,12 +743,23 @@ Decimal objects .. method:: normalize(context=None) - Normalize the number by stripping the rightmost trailing zeros and - converting any result equal to ``Decimal('0')`` to - ``Decimal('0e0')``. Used for producing canonical values for attributes - of an equivalence class. For example, ``Decimal('32.100')`` and - ``Decimal('0.321000e+2')`` both normalize to the equivalent value - ``Decimal('32.1')``. + Used for producing canonical values of an equivalence + class within either the current context or the specified context. + + This has the same semantics as the unary plus operation, except that if + the final result is finite it is reduced to its simplest form, with all + trailing zeros removed and its sign preserved. That is, while the + coefficient is non-zero and a multiple of ten the coefficient is divided + by ten and the exponent is incremented by 1. Otherwise (the coefficient is + zero) the exponent is set to 0. In all cases the sign is unchanged. + + For example, ``Decimal('32.100')`` and ``Decimal('0.321000e+2')`` both + normalize to the equivalent value ``Decimal('32.1')``. + + Note that rounding is applied *before* reducing to simplest form. + + In the latest versions of the specification, this operation is also known + as ``reduce``. .. method:: number_class(context=None) @@ -2078,6 +2089,26 @@ representative: >>> [v.normalize() for v in values] [Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2')] +Q. When does rounding occur in a computation? + +A. It occurs *after* the computation. The philosophy of the decimal +specification is that numbers are considered exact and are created +independent of the current context. They can even have greater +precision than current context. Computations process with those +exact inputs and then rounding (or other context operations) is +applied to the *result* of the computation:: + + >>> getcontext().prec = 5 + >>> pi = Decimal('3.1415926535') # More than 5 digits + >>> pi # All digits are retained + Decimal('3.1415926535') + >>> pi + 0 # Rounded after an addition + Decimal('3.1416') + >>> pi - Decimal('0.00005') # Subtract unrounded numbers, then round + Decimal('3.1415') + >>> pi + 0 - Decimal('0.00005'). # Intermediate values are rounded + Decimal('3.1416') + Q. Some decimal values always print with exponential notation. Is there a way to get a non-exponential representation? diff --git a/Doc/library/devmode.rst b/Doc/library/devmode.rst index 977735990ffe92..80ac13b116c1d2 100644 --- a/Doc/library/devmode.rst +++ b/Doc/library/devmode.rst @@ -16,7 +16,7 @@ setting the :envvar:`PYTHONDEVMODE` environment variable to ``1``. See also :ref:`Python debug build `. Effects of the Python Development Mode -====================================== +-------------------------------------- Enabling the Python Development Mode is similar to the following command, but with additional effects described below:: @@ -107,7 +107,7 @@ value can be read from :data:`sys.flags.dev_mode `. ResourceWarning Example -======================= +----------------------- Example of a script counting the number of lines of the text file specified in the command line:: @@ -171,7 +171,7 @@ application more deterministic and more reliable. Bad file descriptor error example -================================= +--------------------------------- Script displaying the first line of itself:: @@ -198,7 +198,7 @@ descriptor" error when finalizing the file object: .. code-block:: shell-session - $ python script.py + $ python -X dev script.py import os script.py:10: ResourceWarning: unclosed file <_io.TextIOWrapper name='script.py' mode='r' encoding='UTF-8'> main() diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 9b90f1ef23d92c..3b31384edeb54b 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1389,12 +1389,15 @@ iterations of the loop. * ``0x02`` a dictionary of keyword-only parameters' default values * ``0x04`` a tuple of strings containing parameters' annotations * ``0x08`` a tuple containing cells for free variables, making a closure - * the code associated with the function (at ``STACK[-2]``) - * the :term:`qualified name` of the function (at ``STACK[-1]``) + * the code associated with the function (at ``STACK[-1]``) .. versionchanged:: 3.10 Flag value ``0x04`` is a tuple of strings instead of dictionary + .. versionchanged:: 3.11 + Qualified name at ``STACK[-1]`` was removed. + + .. opcode:: BUILD_SLICE (argc) .. index:: pair: built-in function; slice diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 345b64001c1ace..a87a0bd2e7de6b 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -65,6 +65,11 @@ of the new API. *email address* parts. Returns a tuple of that information, unless the parse fails, in which case a 2-tuple of ``('', '')`` is returned. + .. versionchanged:: 3.12 + For security reasons, addresses that were ambiguous and could parse into + multiple different addresses now cause ``('', '')`` to be returned + instead of only one of the *potential* addresses. + .. function:: formataddr(pair, charset='utf-8') @@ -87,7 +92,7 @@ of the new API. This method returns a list of 2-tuples of the form returned by ``parseaddr()``. *fieldvalues* is a sequence of header field values as might be returned by :meth:`Message.get_all `. Here's a simple - example that gets all the recipients of a message:: + example that gets all the recipients of a message: from email.utils import getaddresses @@ -97,6 +102,25 @@ of the new API. resent_ccs = msg.get_all('resent-cc', []) all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs) + When parsing fails for a single fieldvalue, a 2-tuple of ``('', '')`` + is returned in its place. Other errors in parsing the list of + addresses such as a fieldvalue seemingly parsing into multiple + addresses may result in a list containing a single empty 2-tuple + ``[('', '')]`` being returned rather than returning potentially + invalid output. + + Example malformed input parsing: + + .. doctest:: + + >>> from email.utils import getaddresses + >>> getaddresses(['alice@example.com ', 'me@example.com']) + [('', '')] + + .. versionchanged:: 3.12 + The 2-tuple of ``('', '')`` in the returned values when parsing + fails were added as to address a security issue. + .. function:: parsedate(date) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 4c84e5f855431a..d54c49fff6b62a 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -450,7 +450,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: StopAsyncIteration - Must be raised by :meth:`__anext__` method of an + Must be raised by :meth:`~object.__anext__` method of an :term:`asynchronous iterator` object to stop the iteration. .. versionadded:: 3.5 @@ -871,6 +871,8 @@ The following exceptions are used as warning categories; see the .. versionadded:: 3.2 +.. _lib-exception-groups: + Exception groups ---------------- diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3d2bb8efc95d8e..d8091f0b093aab 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -122,7 +122,7 @@ are always available. They are listed here in alphabetical order. Convert an integer number to a binary string prefixed with "0b". The result is a valid Python expression. If *x* is not a Python :class:`int` object, it - has to define an :meth:`__index__` method that returns an integer. Some + has to define an :meth:`~object.__index__` method that returns an integer. Some examples: >>> bin(3) @@ -383,9 +383,9 @@ are always available. They are listed here in alphabetical order. ``0j``. For a general Python object ``x``, ``complex(x)`` delegates to - ``x.__complex__()``. If ``__complex__()`` is not defined then it falls back - to :meth:`__float__`. If ``__float__()`` is not defined then it falls back - to :meth:`__index__`. + ``x.__complex__()``. If :meth:`~object.__complex__` is not defined then it falls back + to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + to :meth:`~object.__index__`. .. note:: @@ -400,8 +400,8 @@ are always available. They are listed here in alphabetical order. Grouping digits with underscores as in code literals is allowed. .. versionchanged:: 3.8 - Falls back to :meth:`__index__` if :meth:`__complex__` and - :meth:`__float__` are not defined. + Falls back to :meth:`~object.__index__` if :meth:`~object.__complex__` and + :meth:`~object.__float__` are not defined. .. function:: delattr(object, name) @@ -681,8 +681,8 @@ are always available. They are listed here in alphabetical order. float, an :exc:`OverflowError` will be raised. For a general Python object ``x``, ``float(x)`` delegates to - ``x.__float__()``. If ``__float__()`` is not defined then it falls back - to :meth:`__index__`. + ``x.__float__()``. If :meth:`~object.__float__` is not defined then it falls back + to :meth:`~object.__index__`. If no argument is given, ``0.0`` is returned. @@ -708,7 +708,7 @@ are always available. They are listed here in alphabetical order. *x* is now a positional-only parameter. .. versionchanged:: 3.8 - Falls back to :meth:`__index__` if :meth:`__float__` is not defined. + Falls back to :meth:`~object.__index__` if :meth:`~object.__float__` is not defined. .. index:: @@ -794,7 +794,7 @@ are always available. They are listed here in alphabetical order. For objects with custom :meth:`__hash__` methods, note that :func:`hash` truncates the return value based on the bit width of the host machine. - See :meth:`__hash__` for details. + See :meth:`__hash__ ` for details. .. function:: help() help(request) @@ -822,7 +822,7 @@ are always available. They are listed here in alphabetical order. Convert an integer number to a lowercase hexadecimal string prefixed with "0x". If *x* is not a Python :class:`int` object, it has to define an - :meth:`__index__` method that returns an integer. Some examples: + :meth:`~object.__index__` method that returns an integer. Some examples: >>> hex(255) '0xff' @@ -893,9 +893,9 @@ are always available. They are listed here in alphabetical order. int(x, base=10) Return an integer object constructed from a number or string *x*, or return - ``0`` if no arguments are given. If *x* defines :meth:`__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__index__`, - it returns ``x.__index__()``. If *x* defines :meth:`__trunc__`, + ``0`` if no arguments are given. If *x* defines :meth:`~object.__int__`, + ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`~object.__index__`, + it returns ``x.__index__()``. If *x* defines :meth:`~object.__trunc__`, it returns ``x.__trunc__()``. For floating point numbers, this truncates towards zero. @@ -932,10 +932,10 @@ are always available. They are listed here in alphabetical order. *x* is now a positional-only parameter. .. versionchanged:: 3.8 - Falls back to :meth:`__index__` if :meth:`__int__` is not defined. + Falls back to :meth:`~object.__index__` if :meth:`~object.__int__` is not defined. .. versionchanged:: 3.11 - The delegation to :meth:`__trunc__` is deprecated. + The delegation to :meth:`~object.__trunc__` is deprecated. .. versionchanged:: 3.11 :class:`int` string inputs and string representations can be limited to @@ -1138,7 +1138,7 @@ are always available. They are listed here in alphabetical order. Convert an integer number to an octal string prefixed with "0o". The result is a valid Python expression. If *x* is not a Python :class:`int` object, it - has to define an :meth:`__index__` method that returns an integer. For + has to define an :meth:`~object.__index__` method that returns an integer. For example: >>> oct(8) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 797870b9d7e260..8102767a43d6dd 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -11,7 +11,7 @@ .. index:: single: message digest, MD5 - single: secure hash algorithm, SHA1, SHA224, SHA256, SHA384, SHA512 + single: secure hash algorithm, SHA1, SHA2, SHA224, SHA256, SHA384, SHA512, SHA3, Shake, Blake2 .. testsetup:: @@ -22,7 +22,8 @@ This module implements a common interface to many different secure hash and message digest algorithms. Included are the FIPS secure hash algorithms SHA1, -SHA224, SHA256, SHA384, and SHA512 (defined in FIPS 180-2) as well as RSA's MD5 +SHA224, SHA256, SHA384, SHA512, (defined in `the FIPS 180-4 standard`_), +the SHA-3 series (defined in `the FIPS 202 standard`_) as well as RSA's MD5 algorithm (defined in internet :rfc:`1321`). The terms "secure hash" and "message digest" are interchangeable. Older algorithms were called message digests. The modern term is secure hash. @@ -32,11 +33,6 @@ digests. The modern term is secure hash. If you want the adler32 or crc32 hash functions, they are available in the :mod:`zlib` module. -.. warning:: - - Some algorithms have known hash collision weaknesses, refer to the "See - also" section at the end. - .. _hash-algorithms: @@ -44,38 +40,43 @@ Hash algorithms --------------- There is one constructor method named for each type of :dfn:`hash`. All return -a hash object with the same simple interface. For example: use :func:`sha256` to -create a SHA-256 hash object. You can now feed this object with :term:`bytes-like -objects ` (normally :class:`bytes`) using the :meth:`update` method. -At any point you can ask it for the :dfn:`digest` of the -concatenation of the data fed to it so far using the :meth:`digest` or -:meth:`hexdigest` methods. - -.. note:: - - For better multithreading performance, the Python :term:`GIL` is released for - data larger than 2047 bytes at object creation or on update. +a hash object with the same simple interface. For example: use :func:`sha256` +to create a SHA-256 hash object. You can now feed this object with +:term:`bytes-like objects ` (normally :class:`bytes`) using +the :meth:`update` method. At any point you can ask it for the +:dfn:`digest` of the concatenation of the data fed to it so far using the +:meth:`digest()` or :meth:`hexdigest()` methods. -.. note:: +To allow multithreading, the Python :term:`GIL` is released while computing a +hash supplied more than 2047 bytes of data at once in its constructor or +:meth:`.update` method. - Feeding string objects into :meth:`update` is not supported, as hashes work - on bytes, not on characters. .. index:: single: OpenSSL; (use in module hashlib) Constructors for hash algorithms that are always present in this module are -:func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, -:func:`sha512`, :func:`blake2b`, and :func:`blake2s`. -:func:`md5` is normally available as well, though it -may be missing or blocked if you are using a rare "FIPS compliant" build of Python. -Additional algorithms may also be available depending upon the OpenSSL -library that Python uses on your platform. On most platforms the +:func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, :func:`sha512`, :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`, -:func:`shake_128`, :func:`shake_256` are also available. +:func:`shake_128`, :func:`shake_256`, :func:`blake2b`, and :func:`blake2s`. +:func:`md5` is normally available as well, though it may be missing or blocked +if you are using a rare "FIPS compliant" build of Python. +These correspond to :data:`algorithms_guaranteed`. + +Additional algorithms may also be available if your Python distribution's +:mod:`hashlib` was linked against a build of OpenSSL that provides others. +Others *are not guaranteed available* on all installations and will only be +accessible by name via :func:`new`. See :data:`algorithms_available`. + +.. warning:: + + Some algorithms have known hash collision weaknesses (including MD5 and + SHA1). Refer to `Attacks on cryptographic hash algorithms`_ and the + `hashlib-seealso`_ section at the end of this document. .. versionadded:: 3.6 SHA3 (Keccak) and SHAKE constructors :func:`sha3_224`, :func:`sha3_256`, - :func:`sha3_384`, :func:`sha3_512`, :func:`shake_128`, :func:`shake_256`. + :func:`sha3_384`, :func:`sha3_512`, :func:`shake_128`, :func:`shake_256` + were added. .. versionadded:: 3.6 :func:`blake2b` and :func:`blake2s` were added. @@ -89,10 +90,19 @@ library that Python uses on your platform. On most platforms the that the hashing algorithm is not used in a security context, e.g. as a non-cryptographic one-way compression function. - Hashlib now uses SHA3 and SHAKE from OpenSSL 1.1.1 and newer. +.. versionchanged:: 3.9 + Hashlib now uses SHA3 and SHAKE from OpenSSL if it provides it. + +.. versionchanged:: 3.12 + For any of the MD5, SHA1, SHA2, or SHA3 algorithms that the linked + OpenSSL does not provide we fall back to a verified implementation from + the `HACL\* project`_. + +Usage +----- -For example, to obtain the digest of the byte string ``b"Nobody inspects the -spammish repetition"``:: +To obtain the digest of the byte string ``b"Nobody inspects the spammish +repetition"``:: >>> import hashlib >>> m = hashlib.sha256() @@ -108,22 +118,42 @@ More condensed: >>> hashlib.sha256(b"Nobody inspects the spammish repetition").hexdigest() '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406' -.. function:: new(name[, data], *, usedforsecurity=True) +Constructors +------------ + +.. function:: new(name[, data], \*, usedforsecurity=True) Is a generic constructor that takes the string *name* of the desired algorithm as its first parameter. It also exists to allow access to the above listed hashes as well as any other algorithms that your OpenSSL - library may offer. The named constructors are much faster than :func:`new` - and should be preferred. + library may offer. -Using :func:`new` with an algorithm provided by OpenSSL: +Using :func:`new` with an algorithm name: >>> h = hashlib.new('sha256') >>> h.update(b"Nobody inspects the spammish repetition") >>> h.hexdigest() '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406' -Hashlib provides the following constant attributes: + +.. function:: md5([, data], \*, usedforsecurity=True) +.. function:: sha1([, data], \*, usedforsecurity=True) +.. function:: sha224([, data], \*, usedforsecurity=True) +.. function:: sha256([, data], \*, usedforsecurity=True) +.. function:: sha384([, data], \*, usedforsecurity=True) +.. function:: sha512([, data], \*, usedforsecurity=True) +.. function:: sha3_224([, data], \*, usedforsecurity=True) +.. function:: sha3_256([, data], \*, usedforsecurity=True) +.. function:: sha3_384([, data], \*, usedforsecurity=True) +.. function:: sha3_512([, data], \*, usedforsecurity=True) + +Named constructors such as these are faster than passing an algorithm name to +:func:`new`. + +Attributes +---------- + +Hashlib provides the following constant module attributes: .. data:: algorithms_guaranteed @@ -144,10 +174,12 @@ Hashlib provides the following constant attributes: .. versionadded:: 3.2 +Hash Objects +------------ + The following values are provided as constant attributes of the hash objects returned by the constructors: - .. data:: hash.digest_size The size of the resulting hash in bytes. @@ -178,11 +210,6 @@ A hash object has the following methods: concatenation of all the arguments: ``m.update(a); m.update(b)`` is equivalent to ``m.update(a+b)``. - .. versionchanged:: 3.1 - The Python GIL is released to allow other threads to run while hash - updates on data larger than 2047 bytes is taking place when using hash - algorithms supplied by OpenSSL. - .. method:: hash.digest() @@ -207,6 +234,9 @@ A hash object has the following methods: SHAKE variable length digests ----------------------------- +.. function:: shake_128([, data], \*, usedforsecurity=True) +.. function:: shake_256([, data], \*, usedforsecurity=True) + The :func:`shake_128` and :func:`shake_256` algorithms provide variable length digests with length_in_bits//2 up to 128 or 256 bits of security. As such, their digest methods require a length. Maximum length is not limited @@ -223,8 +253,13 @@ by the SHAKE algorithm. Like :meth:`digest` except the digest is returned as a string object of double length, containing only hexadecimal digits. This may be used to - exchange the value safely in email or other non-binary environments. + exchange the value in email or other non-binary environments. +Example use: + + >>> h = hashlib.shake_256(b'Nobody inspects the spammish repetition') + >>> h.hexdigest(20) + '44709d6fcb83d92a76dcb0b668c98e1b1d3dafe7' File hashing ------------ @@ -768,12 +803,18 @@ Domain Dedication 1.0 Universal: .. _BLAKE2: https://www.blake2.net .. _HMAC: https://en.wikipedia.org/wiki/Hash-based_message_authentication_code .. _BLAKE: https://web.archive.org/web/20200918190133/https://131002.net/blake/ -.. _SHA-3: https://en.wikipedia.org/wiki/NIST_hash_function_competition +.. _SHA-3: https://en.wikipedia.org/wiki/Secure_Hash_Algorithms .. _ChaCha: https://cr.yp.to/chacha.html .. _pyblake2: https://pythonhosted.org/pyblake2/ .. _NIST-SP-800-132: https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-132.pdf .. _stackexchange pbkdf2 iterations question: https://security.stackexchange.com/questions/3959/recommended-of-iterations-when-using-pbkdf2-sha256/ +.. _Attacks on cryptographic hash algorithms: https://en.wikipedia.org/wiki/Cryptographic_hash_function#Attacks_on_cryptographic_hash_algorithms +.. _the FIPS 180-4 standard: https://csrc.nist.gov/publications/detail/fips/180/4/final +.. _the FIPS 202 standard: https://csrc.nist.gov/publications/detail/fips/202/final +.. _HACL\* project: https://github.com/hacl-star/hacl-star + +.. _hashlib-seealso: .. seealso:: @@ -783,15 +824,18 @@ Domain Dedication 1.0 Universal: Module :mod:`base64` Another way to encode binary hashes for non-binary environments. - https://www.blake2.net - Official BLAKE2 website. + https://nvlpubs.nist.gov/nistpubs/fips/nist.fips.180-4.pdf + The FIPS 180-4 publication on Secure Hash Algorithms. + + https://csrc.nist.gov/publications/detail/fips/202/final + The FIPS 202 publication on the SHA-3 Standard. - https://csrc.nist.gov/csrc/media/publications/fips/180/2/archive/2002-08-01/documents/fips180-2.pdf - The FIPS 180-2 publication on Secure Hash Algorithms. + https://www.blake2.net/ + Official BLAKE2 website. - https://en.wikipedia.org/wiki/Cryptographic_hash_function#Cryptographic_hash_algorithms - Wikipedia article with information on which algorithms have known issues and - what that means regarding their use. + https://en.wikipedia.org/wiki/Cryptographic_hash_function + Wikipedia article with information on which algorithms have known issues + and what that means regarding their use. https://www.ietf.org/rfc/rfc8018.txt PKCS #5: Password-Based Cryptography Specification Version 2.1 diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 45291933d635b9..b9ceab699cef63 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -390,7 +390,7 @@ HTTPConnection Objects Returns a dictionary with the headers of the response received from the proxy server to the CONNECT request. - If the CONNECT request was not sent, the method returns an empty dictionary. + If the CONNECT request was not sent, the method returns ``None``. .. versionadded:: 3.12 diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 56d6599798af20..730736bbb59ed9 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1045,13 +1045,14 @@ The following recipes have a more mathematical flavor: def factor(n): "Prime factors of n." # factor(99) --> 3 3 11 + # factor(1_000_000_000_000_007) --> 47 59 360620266859 + # factor(1_000_000_000_000_403) --> 1000000000000403 for prime in sieve(math.isqrt(n) + 1): while True: - quotient, remainder = divmod(n, prime) - if remainder: + if n % prime: break yield prime - n = quotient + n //= prime if n == 1: return if n > 1: @@ -1085,8 +1086,8 @@ The following recipes have a more mathematical flavor: kernel = tuple(kernel)[::-1] n = len(kernel) padded_signal = chain(repeat(0, n-1), signal, repeat(0, n-1)) - for window in sliding_window(padded_signal, n): - yield math.sumprod(kernel, window) + windowed_signal = sliding_window(padded_signal, n) + return map(math.sumprod, repeat(kernel), windowed_signal) def polynomial_from_roots(roots): """Compute a polynomial's coefficients from its roots. @@ -1352,6 +1353,12 @@ The following recipes have a more mathematical flavor: >>> set(sieve(10_000)).isdisjoint(carmichael) True + >>> list(factor(99)) # Code example 1 + [3, 3, 11] + >>> list(factor(1_000_000_000_000_007)) # Code example 2 + [47, 59, 360620266859] + >>> list(factor(1_000_000_000_000_403)) # Code example 3 + [1000000000000403] >>> list(factor(0)) [] >>> list(factor(1)) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index ef58dd09423640..5383614575c213 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -9,11 +9,6 @@ **Source code:** :source:`Lib/json/__init__.py` -.. testsetup:: * - - import json - from json import AttrDict - -------------- `JSON (JavaScript Object Notation) `_, specified by @@ -548,44 +543,6 @@ Exceptions .. versionadded:: 3.5 -.. class:: AttrDict(**kwargs) - AttrDict(mapping, **kwargs) - AttrDict(iterable, **kwargs) - - Subclass of :class:`dict` that also supports attribute style dotted access. - - This class is intended for use with the :attr:`object_hook` in - :func:`json.load` and :func:`json.loads`: - - .. doctest:: - - >>> json_string = '{"mercury": 88, "venus": 225, "earth": 365, "mars": 687}' - >>> orbital_period = json.loads(json_string, object_hook=AttrDict) - >>> orbital_period['earth'] # Dict style lookup - 365 - >>> orbital_period.earth # Attribute style lookup - 365 - >>> orbital_period.keys() # All dict methods are present - dict_keys(['mercury', 'venus', 'earth', 'mars']) - - Attribute style access only works for keys that are valid attribute - names. In contrast, dictionary style access works for all keys. For - example, ``d.two words`` contains a space and is not syntactically - valid Python, so ``d["two words"]`` should be used instead. - - If a key has the same name as a dictionary method, then a dictionary - lookup finds the key and an attribute lookup finds the method: - - .. doctest:: - - >>> d = AttrDict(items=50) - >>> d['items'] # Lookup the key - 50 - >>> d.items() # Call the method - dict_items([('items', 50)]) - - .. versionadded:: 3.12 - Standard Compliance and Interoperability ---------------------------------------- diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 5c02d8bc8835bf..0cff3817452364 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -42,8 +42,8 @@ on the command-line, for example:: --file=outfile -q As it parses the command line, :mod:`optparse` sets attributes of the -``options`` object returned by :meth:`parse_args` based on user-supplied -command-line values. When :meth:`parse_args` returns from parsing this command +``options`` object returned by :meth:`~OptionParser.parse_args` based on user-supplied +command-line values. When :meth:`~OptionParser.parse_args` returns from parsing this command line, ``options.filename`` will be ``"outfile"`` and ``options.verbose`` will be ``False``. :mod:`optparse` supports both long and short options, allows short options to be merged together, and allows options to be associated with their @@ -285,10 +285,10 @@ program's command line:: (options, args) = parser.parse_args() -(If you like, you can pass a custom argument list to :meth:`parse_args`, but +(If you like, you can pass a custom argument list to :meth:`~OptionParser.parse_args`, but that's rarely necessary: by default it uses ``sys.argv[1:]``.) -:meth:`parse_args` returns two values: +:meth:`~OptionParser.parse_args` returns two values: * ``options``, an object containing values for all of your options---e.g. if ``--file`` takes a single string argument, then ``options.file`` will be the @@ -339,7 +339,7 @@ Now let's make up a fake command line and ask :mod:`optparse` to parse it:: When :mod:`optparse` sees the option string ``-f``, it consumes the next argument, ``foo.txt``, and stores it in ``options.filename``. So, after this -call to :meth:`parse_args`, ``options.filename`` is ``"foo.txt"``. +call to :meth:`~OptionParser.parse_args`, ``options.filename`` is ``"foo.txt"``. Some other option types supported by :mod:`optparse` are ``int`` and ``float``. Here's an option that expects an integer argument:: @@ -453,7 +453,8 @@ Again, the default value for ``verbose`` will be ``True``: the last default value supplied for any particular destination is the one that counts. A clearer way to specify default values is the :meth:`set_defaults` method of -OptionParser, which you can call at any time before calling :meth:`parse_args`:: +OptionParser, which you can call at any time before calling +:meth:`~OptionParser.parse_args`:: parser.set_defaults(verbose=True) parser.add_option(...) @@ -1338,35 +1339,37 @@ Parsing arguments ^^^^^^^^^^^^^^^^^ The whole point of creating and populating an OptionParser is to call its -:meth:`parse_args` method:: +:meth:`~OptionParser.parse_args` method. - (options, args) = parser.parse_args(args=None, values=None) +.. method:: OptionParser.parse_args(args=None, values=None) -where the input parameters are + Parse the command-line options found in *args*. -``args`` - the list of arguments to process (default: ``sys.argv[1:]``) + The input parameters are -``values`` - an :class:`optparse.Values` object to store option arguments in (default: a - new instance of :class:`Values`) -- if you give an existing object, the - option defaults will not be initialized on it + ``args`` + the list of arguments to process (default: ``sys.argv[1:]``) -and the return values are + ``values`` + an :class:`Values` object to store option arguments in (default: a + new instance of :class:`Values`) -- if you give an existing object, the + option defaults will not be initialized on it -``options`` - the same object that was passed in as ``values``, or the optparse.Values - instance created by :mod:`optparse` + and the return value is a pair ``(options, args)`` where -``args`` - the leftover positional arguments after all options have been processed + ``options`` + the same object that was passed in as *values*, or the ``optparse.Values`` + instance created by :mod:`optparse` + + ``args`` + the leftover positional arguments after all options have been processed The most common usage is to supply neither keyword argument. If you supply ``values``, it will be modified with repeated :func:`setattr` calls (roughly one for every option argument stored to an option destination) and returned by -:meth:`parse_args`. +:meth:`~OptionParser.parse_args`. -If :meth:`parse_args` encounters any errors in the argument list, it calls the +If :meth:`~OptionParser.parse_args` encounters any errors in the argument list, it calls the OptionParser's :meth:`error` method with an appropriate end-user error message. This ultimately terminates your process with an exit status of 2 (the traditional Unix exit status for command-line errors). @@ -1661,7 +1664,7 @@ where the current list of leftover arguments, ie. arguments that have been consumed but are neither options nor option arguments. Feel free to modify ``parser.largs``, e.g. by adding more arguments to it. (This list will - become ``args``, the second return value of :meth:`parse_args`.) + become ``args``, the second return value of :meth:`~OptionParser.parse_args`.) ``parser.rargs`` the current list of remaining arguments, ie. with ``opt_str`` and diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 39b9eb84e7e068..2fb4dc42f46feb 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -585,8 +585,8 @@ Pure paths provide the following methods and properties: Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - .. versionadded:: 3.12 - The *case_sensitive* argument. + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. .. method:: PurePath.relative_to(other, walk_up=False) @@ -627,8 +627,8 @@ Pure paths provide the following methods and properties: are present in the path; call :meth:`~Path.resolve` first if necessary to resolve symlinks. - .. versionadded:: 3.12 - The *walk_up* argument (old behavior is the same as ``walk_up=False``). + .. versionchanged:: 3.12 + The *walk_up* parameter was added (old behavior is the same as ``walk_up=False``). .. deprecated-removed:: 3.12 3.14 @@ -928,8 +928,9 @@ call fails (for example because the path doesn't exist). Return only directories if *pattern* ends with a pathname components separator (:data:`~os.sep` or :data:`~os.altsep`). - .. versionadded:: 3.12 - The *case_sensitive* argument. + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + .. method:: Path.group() @@ -1313,8 +1314,8 @@ call fails (for example because the path doesn't exist). infinite loop is encountered along the resolution path, :exc:`RuntimeError` is raised. - .. versionadded:: 3.6 - The *strict* argument (pre-3.6 behavior is strict). + .. versionchanged:: 3.6 + The *strict* parameter was added (pre-3.6 behavior is strict). .. method:: Path.rglob(pattern, *, case_sensitive=None) @@ -1340,8 +1341,9 @@ call fails (for example because the path doesn't exist). Return only directories if *pattern* ends with a pathname components separator (:data:`~os.sep` or :data:`~os.altsep`). - .. versionadded:: 3.12 - The *case_sensitive* argument. + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + .. method:: Path.rmdir() diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index 0bad51833aae13..f94833ad5331a9 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -30,12 +30,6 @@ The :mod:`shlex` module defines the following functions: in POSIX mode by default, but uses non-POSIX mode if the *posix* argument is false. - .. note:: - - Since the :func:`split` function instantiates a :class:`~shlex.shlex` - instance, passing ``None`` for *s* will read the string to split from - standard input. - .. versionchanged:: 3.12 Passing ``None`` for *s* argument now raises an exception, rather than reading :data:`sys.stdin`. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e7129fb3e4de6d..2e2c28b1e5fe73 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -266,8 +266,9 @@ Module functions :param database: The path to the database file to be opened. - Pass ``":memory:"`` to open a connection to a database that is - in RAM instead of on disk. + You can pass ``":memory:"`` to create an `SQLite database existing only + in memory `_, and open a connection + to it. :type database: :term:`path-like object` :param float timeout: @@ -413,15 +414,15 @@ Module functions .. function:: register_adapter(type, adapter, /) - Register an *adapter* callable to adapt the Python type *type* into an - SQLite type. + Register an *adapter* :term:`callable` to adapt the Python type *type* + into an SQLite type. The adapter is called with a Python object of type *type* as its sole argument, and must return a value of a :ref:`type that SQLite natively understands `. .. function:: register_converter(typename, converter, /) - Register the *converter* callable to convert SQLite objects of type + Register the *converter* :term:`callable` to convert SQLite objects of type *typename* into a Python object of a specific type. The converter is invoked for all SQLite values of type *typename*; it is passed a :class:`bytes` object and should return an object of the @@ -484,7 +485,7 @@ Module constants SQLITE_DENY SQLITE_IGNORE - Flags that should be returned by the *authorizer_callback* callable + Flags that should be returned by the *authorizer_callback* :term:`callable` passed to :meth:`Connection.set_authorizer`, to indicate whether: * Access is allowed (:const:`!SQLITE_OK`), @@ -629,8 +630,8 @@ Connection objects Create and return a :class:`Cursor` object. The cursor method accepts a single optional parameter *factory*. If - supplied, this must be a callable returning an instance of :class:`Cursor` - or its subclasses. + supplied, this must be a :term:`callable` returning + an instance of :class:`Cursor` or its subclasses. .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") @@ -723,7 +724,7 @@ Connection objects If ``-1``, it may take any number of arguments. :param func: - A callable that is called when the SQL function is invoked. + A :term:`callable` that is called when the SQL function is invoked. The callable must return :ref:`a type natively supported by SQLite `. Set to ``None`` to remove an existing SQL function. @@ -948,9 +949,10 @@ Connection objects .. method:: set_authorizer(authorizer_callback) - Register callable *authorizer_callback* to be invoked for each attempt to - access a column of a table in the database. The callback should return - one of :const:`SQLITE_OK`, :const:`SQLITE_DENY`, or :const:`SQLITE_IGNORE` + Register :term:`callable` *authorizer_callback* to be invoked + for each attempt to access a column of a table in the database. + The callback should return one of :const:`SQLITE_OK`, + :const:`SQLITE_DENY`, or :const:`SQLITE_IGNORE` to signal how access to the column should be handled by the underlying SQLite library. @@ -973,7 +975,7 @@ Connection objects .. method:: set_progress_handler(progress_handler, n) - Register callable *progress_handler* to be invoked for every *n* + Register :term:`callable` *progress_handler* to be invoked for every *n* instructions of the SQLite virtual machine. This is useful if you want to get called from SQLite during long-running operations, for example to update a GUI. @@ -988,8 +990,8 @@ Connection objects .. method:: set_trace_callback(trace_callback) - Register callable *trace_callback* to be invoked for each SQL statement - that is actually executed by the SQLite backend. + Register :term:`callable` *trace_callback* to be invoked + for each SQL statement that is actually executed by the SQLite backend. The only argument passed to the callback is the statement (as :class:`str`) that is being executed. The return value of the callback is @@ -1139,8 +1141,8 @@ Connection objects Defaults to ``-1``. :param progress: - If set to a callable, it is invoked with three integer arguments for - every backup iteration: + If set to a :term:`callable`, + it is invoked with three integer arguments for every backup iteration: the *status* of the last iteration, the *remaining* number of pages still to be copied, and the *total* number of pages. @@ -1404,8 +1406,8 @@ Connection objects .. attribute:: text_factory - A callable that accepts a :class:`bytes` parameter and returns a text - representation of it. + A :term:`callable` that accepts a :class:`bytes` parameter + and returns a text representation of it. The callable is invoked for SQLite values with the ``TEXT`` data type. By default, this attribute is set to :class:`str`. If you want to return ``bytes`` instead, set *text_factory* to ``bytes``. @@ -2521,6 +2523,13 @@ Queries now return :class:`!Row` objects: >>> row["RADIUS"] # Column names are case-insensitive. 6378 +.. note:: + + The ``FROM`` clause can be omitted in the ``SELECT`` statement, as in the + above example. In such cases, SQLite returns a single row with columns + defined by expressions, e.g. literals, with the given aliases + ``expr AS alias``. + You can create a custom :attr:`~Cursor.row_factory` that returns each row as a :class:`dict`, with column names mapped to values: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 0caa725f75e642..092f2719379e8a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -52,7 +52,7 @@ objects considered false: single: None (Built-in object) single: False (Built-in object) -* constants defined to be false: ``None`` and ``False``. +* constants defined to be false: ``None`` and ``False`` * zero of any numeric type: ``0``, ``0.0``, ``0j``, ``Decimal(0)``, ``Fraction(0, 1)`` @@ -3950,7 +3950,7 @@ copying. >>> m = memoryview(bytearray(b'abc')) >>> mm = m.toreadonly() >>> mm.tolist() - [89, 98, 99] + [97, 98, 99] >>> mm[0] = 42 Traceback (most recent call last): File "", line 1, in @@ -4006,6 +4006,7 @@ copying. :mod:`struct` syntax. One of the formats must be a byte format ('B', 'b' or 'c'). The byte length of the result must be the same as the original length. + Note that all byte lengths may depend on the operating system. Cast 1D/long to 1D/unsigned bytes:: @@ -4036,8 +4037,8 @@ copying. >>> x = memoryview(b) >>> x[0] = b'a' Traceback (most recent call last): - File "", line 1, in - ValueError: memoryview: invalid value for format "B" + ... + TypeError: memoryview: invalid type for format 'B' >>> y = x.cast('c') >>> y[0] = b'a' >>> b @@ -4786,10 +4787,10 @@ An example of dictionary view usage:: >>> # set operations >>> keys & {'eggs', 'bacon', 'salad'} {'bacon'} - >>> keys ^ {'sausage', 'juice'} - {'juice', 'sausage', 'bacon', 'spam'} - >>> keys | ['juice', 'juice', 'juice'] - {'juice', 'sausage', 'bacon', 'spam', 'eggs'} + >>> keys ^ {'sausage', 'juice'} == {'juice', 'sausage', 'bacon', 'spam'} + True + >>> keys | ['juice', 'juice', 'juice'] == {'bacon', 'spam', 'juice'} + True >>> # get back a read-only proxy for the original dictionary >>> values.mapping @@ -4996,8 +4997,8 @@ exception to disallow mistakes like ``dict[str][str]``:: >>> dict[str][str] Traceback (most recent call last): - File "", line 1, in - TypeError: There are no type variables left in dict[str] + ... + TypeError: dict[str] is not a generic class However, such expressions are valid when :ref:`type variables ` are used. The index must have as many elements as there are type variable items @@ -5203,13 +5204,15 @@ enables cleaner type hinting syntax compared to :data:`typing.Union`. >>> isinstance("", int | str) True - However, union objects containing :ref:`parameterized generics - ` cannot be used:: + However, :ref:`parameterized generics ` in + union objects cannot be checked:: - >>> isinstance(1, int | list[int]) + >>> isinstance(1, int | list[int]) # short-circuit evaluation + True + >>> isinstance([1], int | list[int]) Traceback (most recent call last): - File "", line 1, in - TypeError: isinstance() argument 2 cannot contain a parameterized generic + ... + TypeError: isinstance() argument 2 cannot be a parameterized generic The user-exposed type for the union object can be accessed from :data:`types.UnionType` and used for :func:`isinstance` checks. An object cannot be @@ -5512,7 +5515,7 @@ types, where they are relevant. Some of these are not reported by the definition order. Example:: >>> int.__subclasses__() - [] + [, , , ] .. _int_max_str_digits: @@ -5548,7 +5551,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> _ = int('2' * 5432) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit >>> i = int('2' * 4300) >>> len(str(i)) 4300 @@ -5556,7 +5559,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> len(str(i_squared)) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit >>> len(hex(i_squared)) 7144 >>> assert int(hex(i_squared), base=16) == i*i # Hexadecimal is unlimited. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 78fd6e397ae635..6d2739b4557fbf 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -266,11 +266,11 @@ Notes: (2) When attempting to pack a non-integer using any of the integer conversion - codes, if the non-integer has a :meth:`__index__` method then that method is + codes, if the non-integer has a :meth:`~object.__index__` method then that method is called to convert the argument to an integer before packing. .. versionchanged:: 3.2 - Added use of the :meth:`__index__` method for non-integers. + Added use of the :meth:`~object.__index__` method for non-integers. (3) The ``'n'`` and ``'N'`` conversion codes are only available for the native diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 891af1bcf7edff..6a5947b788fdae 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -903,7 +903,7 @@ can be: path to where the archive is extracted (i.e. the same path is used for all members):: - filter(/, member: TarInfo, path: str) -> TarInfo | None + filter(member: TarInfo, path: str, /) -> TarInfo | None The callable is called just before each member is extracted, so it can take the current state of the disk into account. @@ -923,13 +923,13 @@ Default named filters The pre-defined, named filters are available as functions, so they can be reused in custom filters: -.. function:: fully_trusted_filter(/, member, path) +.. function:: fully_trusted_filter(member, path) Return *member* unchanged. This implements the ``'fully_trusted'`` filter. -.. function:: tar_filter(/, member, path) +.. function:: tar_filter(member, path) Implements the ``'tar'`` filter. @@ -946,7 +946,7 @@ reused in custom filters: Return the modified ``TarInfo`` member. -.. function:: data_filter(/, member, path) +.. function:: data_filter(member, path) Implements the ``'data'`` filter. In addition to what ``tar_filter`` does: diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 32ab565aba0c08..a559e0a2eb3dad 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -27,11 +27,11 @@ can be used to compare three different expressions: .. code-block:: shell-session - $ python -m timeit '"-".join(str(n) for n in range(100))' + $ python -m timeit "'-'.join(str(n) for n in range(100))" 10000 loops, best of 5: 30.2 usec per loop - $ python -m timeit '"-".join([str(n) for n in range(100)])' + $ python -m timeit "'-'.join([str(n) for n in range(100)])" 10000 loops, best of 5: 27.5 usec per loop - $ python -m timeit '"-".join(map(str, range(100)))' + $ python -m timeit "'-'.join(map(str, range(100)))" 10000 loops, best of 5: 23.2 usec per loop This can be achieved from the :ref:`python-interface` with:: @@ -277,9 +277,9 @@ It is possible to provide a setup statement that is executed only once at the be .. code-block:: shell-session - $ python -m timeit -s 'text = "sample string"; char = "g"' 'char in text' + $ python -m timeit -s "text = 'sample string'; char = 'g'" "char in text" 5000000 loops, best of 5: 0.0877 usec per loop - $ python -m timeit -s 'text = "sample string"; char = "g"' 'text.find(char)' + $ python -m timeit -s "text = 'sample string'; char = 'g'" "text.find(char)" 1000000 loops, best of 5: 0.342 usec per loop In the output, there are three fields. The loop count, which tells you how many @@ -313,14 +313,14 @@ to test for missing and present object attributes: .. code-block:: shell-session - $ python -m timeit 'try:' ' str.__bool__' 'except AttributeError:' ' pass' + $ python -m timeit "try:" " str.__bool__" "except AttributeError:" " pass" 20000 loops, best of 5: 15.7 usec per loop - $ python -m timeit 'if hasattr(str, "__bool__"): pass' + $ python -m timeit "if hasattr(str, '__bool__'): pass" 50000 loops, best of 5: 4.26 usec per loop - $ python -m timeit 'try:' ' int.__bool__' 'except AttributeError:' ' pass' + $ python -m timeit "try:" " int.__bool__" "except AttributeError:" " pass" 200000 loops, best of 5: 1.43 usec per loop - $ python -m timeit 'if hasattr(int, "__bool__"): pass' + $ python -m timeit "if hasattr(int, '__bool__'): pass" 100000 loops, best of 5: 2.23 usec per loop :: diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index caf4a53006f5e7..e1acbd83a41730 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -23,10 +23,9 @@ -------------- -This module provides runtime support for type hints. The most fundamental -support consists of the types :data:`Any`, :data:`Union`, :data:`Callable`, -:class:`TypeVar`, and :class:`Generic`. For a specification, please see -:pep:`484`. For a simplified introduction to type hints, see :pep:`483`. +This module provides runtime support for type hints. For the original +specification of the typing system, see :pep:`484`. For a simplified +introduction to type hints, see :pep:`483`. The function below takes and returns a string and is annotated as follows:: @@ -47,16 +46,18 @@ For a summary of deprecated features and a deprecation timeline, please see .. seealso:: - For a quick overview of type hints, refer to - `this cheat sheet `_. + `"Typing cheat sheet" `_ + A quick overview of type hints (hosted at the mypy docs) - The "Type System Reference" section of https://mypy.readthedocs.io/ -- since - the Python typing system is standardised via PEPs, this reference should - broadly apply to most Python type checkers, although some parts may still be - specific to mypy. + "Type System Reference" section of `the mypy docs `_ + The Python typing system is standardised via PEPs, so this reference + should broadly apply to most Python type checkers. (Some parts may still + be specific to mypy.) - The documentation at https://typing.readthedocs.io/ serves as useful reference - for type system features, useful typing related tools and typing best practices. + `"Static Typing with Python" `_ + Type-checker-agnostic documentation written by the community detailing + type system features, useful typing related tools and typing best + practices. .. _relevant-peps: @@ -257,18 +258,21 @@ See :pep:`484` for more details. The performance of calling ``NewType`` has been restored to its level in Python 3.9. +.. _annotating-callables: -Callable -======== +Annotating callable objects +=========================== -Frameworks expecting callback functions of specific signatures might be -type hinted using ``Callable[[Arg1Type, Arg2Type], ReturnType]``. +Functions -- or other :term:`callable` objects -- can be annotated using +:class:`collections.abc.Callable` or :data:`typing.Callable`. +``Callable[[int], str]`` signifies a function that takes a single parameter +of type :class:`int` and returns a :class:`str`. For example: .. testcode:: - from collections.abc import Callable + from collections.abc import Callable, Awaitable def feeder(get_next_item: Callable[[], str]) -> None: ... # Body @@ -282,9 +286,49 @@ For example: callback: Callable[[str], Awaitable[None]] = on_update -It is possible to declare the return type of a callable without specifying -the call signature by substituting a literal ellipsis -for the list of arguments in the type hint: ``Callable[..., ReturnType]``. +The subscription syntax must always be used with exactly two values: the +argument list and the return type. The argument list must be a list of types, +a :class:`ParamSpec`, :data:`Concatenate`, or an ellipsis. The return type must +be a single type. + +If a literal ellipsis ``...`` is given as the argument list, it indicates that +a callable with any arbitrary parameter list would be acceptable: + +.. testcode:: + + def concat(x: str, y: str) -> str: + return x + y + + x: Callable[..., str] + x = str # OK + x = concat # Also OK + +``Callable`` cannot express complex signatures such as functions that take a +variadic number of arguments, :func:`overloaded functions `, or +functions that have keyword-only parameters. However, these signatures can be +expressed by defining a :class:`Protocol` class with a +:meth:`~object.__call__` method: + +.. testcode:: + + from collections.abc import Iterable + from typing import Protocol + + class Combiner(Protocol): + def __call__(self, *vals: bytes, maxlen: int | None = None) -> list[bytes]: ... + + def batch_proc(data: Iterable[bytes], cb_results: Combiner) -> bytes: + for item in data: + ... + + def good_cb(*vals: bytes, maxlen: int | None = None) -> list[bytes]: + ... + def bad_cb(*vals: bytes, maxitems: int | None) -> list[bytes]: + ... + + batch_proc([], good_cb) # OK + batch_proc([], bad_cb) # Error! Argument 2 has incompatible type because of + # different name and kind in the callback Callables which take other callables as arguments may indicate that their parameter types are dependent on each other using :class:`ParamSpec`. @@ -345,6 +389,114 @@ Or by using the :class:`TypeVar` factory directly:: .. versionchanged:: 3.12 Syntactic support for generics is new in Python 3.12. +.. _annotating-tuples: + +Annotating tuples +================= + +For most containers in Python, the typing system assumes that all elements in +the container will be of the same type. For example:: + + from collections.abc import Mapping + + # Type checker will infer that all elements in ``x`` are meant to be ints + x: list[int] = [] + + # Type checker error: ``list`` only accepts a single type argument: + y: list[int, str] = [1, 'foo'] + + # Type checker will infer that all keys in ``z`` are meant to be strings, + # and that all values in ``z`` are meant to be either strings or ints + z: Mapping[str, str | int] = {} + +:class:`list` only accepts one type argument, so a type checker would emit an +error on the ``y`` assignment above. Similarly, +:class:`~collections.abc.Mapping` only accepts two type arguments: the first +indicates the type of the keys, and the second indicates the type of the +values. + +Unlike most other Python containers, however, it is common in idiomatic Python +code for tuples to have elements which are not all of the same type. For this +reason, tuples are special-cased in Python's typing system. :class:`tuple` +accepts *any number* of type arguments:: + + # OK: ``x`` is assigned to a tuple of length 1 where the sole element is an int + x: tuple[int] = (5,) + + # OK: ``y`` is assigned to a tuple of length 2; + # element 1 is an int, element 2 is a str + y: tuple[int, str] = (5, "foo") + + # Error: the type annotation indicates a tuple of length 1, + # but ``z`` has been assigned to a tuple of length 3 + z: tuple[int] = (1, 2, 3) + +To denote a tuple which could be of *any* length, and in which all elements are +of the same type ``T``, use ``tuple[T, ...]``. To denote an empty tuple, use +``tuple[()]``. Using plain ``tuple`` as an annotation is equivalent to using +``tuple[Any, ...]``:: + + x: tuple[int, ...] = (1, 2) + # These reassignments are OK: ``tuple[int, ...]`` indicates x can be of any length + x = (1, 2, 3) + x = () + # This reassignment is an error: all elements in ``x`` must be ints + x = ("foo", "bar") + + # ``y`` can only ever be assigned to an empty tuple + y: tuple[()] = () + + z: tuple = ("foo", "bar") + # These reassignments are OK: plain ``tuple`` is equivalent to ``tuple[Any, ...]`` + z = (1, 2, 3) + z = () + +.. _type-of-class-objects: + +The type of class objects +========================= + +A variable annotated with ``C`` may accept a value of type ``C``. In +contrast, a variable annotated with ``type[C]`` (or +:class:`typing.Type[C] `) may accept values that are classes +themselves -- specifically, it will accept the *class object* of ``C``. For +example:: + + a = 3 # Has type ``int``` + b = int # Has type ``type[int]`` + c = type(a) # Also has type ``type[int]`` + +Note that ``type[C]`` is covariant:: + + class User: ... + class ProUser(User): ... + class TeamUser(User): ... + + def make_new_user(user_class: type[User]) -> User: + # ... + return user_class() + + make_new_user(User) # OK + make_new_user(ProUser) # Also OK: ``type[ProUser]`` is a subtype of ``type[User]`` + make_new_user(TeamUser) # Still fine + make_new_user(User()) # Error: expected ``type[User]`` but got ``User`` + make_new_user(int) # Error: ``type[int]`` is not a subtype of ``type[User]`` + +The only legal parameters for :class:`type` are classes, :data:`Any`, +:ref:`type variables `, and unions of any of these types. +For example:: + + def new_non_team_user(user_class: type[BasicUser | ProUser]): ... + + new_non_team_user(BasicUser) # OK + new_non_team_user(ProUser) # OK + new_non_team_user(TeamUser) # Error: ``type[TeamUser]`` is not a subtype + # of ``type[BasicUser | ProUser]`` + new_non_team_user(User) # Also an error + +``type[Any]`` is equivalent to :class:`type`, which is the root of Python's +:ref:`metaclass hierarchy `. + .. _user-defined-generics: User-defined generic types @@ -654,25 +806,7 @@ can define new custom protocols to fully enjoy structural subtyping Module contents =============== -The module defines the following classes, functions and decorators. - -.. note:: - - This module defines several deprecated aliases to pre-existing - standard library classes. These were originally included in the typing - module in order to support parameterizing these generic classes using ``[]``. - However, the aliases became redundant in Python 3.9 when the - corresponding pre-existing classes were enhanced to support ``[]``. - - The redundant types are deprecated as of Python 3.9 but no - deprecation warnings are issued by the interpreter. - It is expected that type checkers will flag the deprecated types - when the checked program targets Python 3.9 or newer. - - The deprecated types will be removed from the :mod:`typing` module - no sooner than the first Python version released 5 years after the release of Python 3.9.0. - See details in :pep:`585`—*Type Hinting Generics In Standard Collections*. - +The ``typing`` module defines the following classes, functions and decorators. Special typing primitives ------------------------- @@ -680,7 +814,8 @@ Special typing primitives Special types """"""""""""" -These can be used as types in annotations and do not support ``[]``. +These can be used as types in annotations. They do not support subscription +using ``[]``. .. data:: Any @@ -890,27 +1025,8 @@ These can be used as types in annotations and do not support ``[]``. Special forms """"""""""""" -These can be used as types in annotations using ``[]``, each having a unique syntax. - -.. data:: Tuple - - Deprecated alias for :class:`tuple`. - - ``Tuple[X, Y]`` is the type of a tuple of two items - with the first item of type X and the second of type Y. The type of - the empty tuple can be written as ``Tuple[()]``. - - Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding - to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple - of an int, a float and a string. - - To specify a variable-length tuple of homogeneous type, - use literal ellipsis, e.g. ``Tuple[int, ...]``. A plain ``Tuple`` annotation - is equivalent to ``tuple``, ``Tuple[Any, ...]``, or ``tuple[Any, ...]``. - - .. deprecated:: 3.9 - :class:`builtins.tuple ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +These can be used as types in annotations. They all support subscription using +``[]``, but each has a unique syntax. .. data:: Union @@ -970,56 +1086,16 @@ These can be used as types in annotations using ``[]``, each having a unique syn Optional can now be written as ``X | None``. See :ref:`union type expressions`. -.. data:: Callable - - Deprecated alias to :class:`collections.abc.Callable`. - - ``Callable[[int], str]`` signifies a function that takes a single parameter - of type :class:`int` and returns a :class:`str`. - - The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types, a :class:`ParamSpec`, :data:`Concatenate`, - or an ellipsis. The return type must be a single type. - - There is no syntax to indicate optional or keyword arguments; - such function types are rarely used as callback types. - ``Callable[..., ReturnType]`` (literal ellipsis) can be used to - type hint a callable taking any number of arguments and returning - ``ReturnType``. A plain :data:`Callable` is equivalent to - ``Callable[..., Any]``, and in turn to - :class:`collections.abc.Callable`. - - Callables which take other callables as arguments may indicate that their - parameter types are dependent on each other using :class:`ParamSpec`. - Additionally, if that callable adds or removes arguments from other - callables, the :data:`Concatenate` operator may be used. They - take the form ``Callable[ParamSpecVariable, ReturnType]`` and - ``Callable[Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable], ReturnType]`` - respectively. - - .. deprecated:: 3.9 - :class:`collections.abc.Callable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - - .. versionchanged:: 3.10 - ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. - See :pep:`612` for more details. - - .. seealso:: - The documentation for :class:`ParamSpec` and :class:`Concatenate` provide - examples of usage with ``Callable``. - .. data:: Concatenate Special form for annotating higher-order functions. - ``Concatenate`` can be used in conjunction with :data:`Callable` and + ``Concatenate`` can be used in conjunction with :ref:`Callable ` and :class:`ParamSpec` to annotate a higher-order callable which adds, removes, or transforms parameters of another callable. Usage is in the form ``Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable]``. ``Concatenate`` - is currently only valid when used as the first argument to a :data:`Callable`. + is currently only valid when used as the first argument to a :ref:`Callable `. The last parameter to ``Concatenate`` must be a :class:`ParamSpec` or ellipsis (``...``). @@ -1063,57 +1139,9 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. seealso:: * :pep:`612` -- Parameter Specification Variables (the PEP which introduced - ``ParamSpec`` and ``Concatenate``). - * :class:`ParamSpec` and :class:`Callable`. - - -.. class:: Type(Generic[CT_co]) - - Deprecated alias to :class:`type`. - - A variable annotated with ``C`` may accept a value of type ``C``. In - contrast, a variable annotated with ``type[C]`` or ``Type[C]`` may accept values that are - classes themselves -- specifically, it will accept the *class object* of - ``C``. For example:: - - a = 3 # Has type 'int' - b = int # Has type 'Type[int]' - c = type(a) # Also has type 'Type[int]' - - Note that ``Type[C]`` is covariant:: - - class User: ... - class BasicUser(User): ... - class ProUser(User): ... - class TeamUser(User): ... - - # Accepts User, BasicUser, ProUser, TeamUser, ... - def make_new_user(user_class: Type[User]) -> User: - # ... - return user_class() - - The fact that ``Type[C]`` is covariant implies that all subclasses of - ``C`` should implement the same constructor signature and class method - signatures as ``C``. The type checker should flag violations of this, - but should also allow constructor calls in subclasses that match the - constructor calls in the indicated base class. How the type checker is - required to handle this particular case may change in future revisions of - :pep:`484`. - - The only legal parameters for :class:`Type` are classes, :data:`Any`, - :ref:`type variables `, and unions of any of these types. - For example:: - - def new_non_team_user(user_class: Type[BasicUser | ProUser]): ... - - ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent - to ``type``, which is the root of Python's metaclass hierarchy. - - .. versionadded:: 3.5.2 - - .. deprecated:: 3.9 - :class:`builtins.type ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + ``ParamSpec`` and ``Concatenate``) + * :class:`ParamSpec` + * :ref:`annotating-callables` .. data:: Literal @@ -1471,7 +1499,8 @@ These can be used as types in annotations using ``[]``, each having a unique syn Building generic types and type aliases """"""""""""""""""""""""""""""""""""""" -The following objects are not used directly in annotations. Instead, they are building blocks +The following classes should not be used directly as annotations. +Their intended purpose is to be building blocks for creating generic types and type aliases. These objects can be created through special syntax @@ -1868,8 +1897,9 @@ without the dedicated syntax, as documented below. .. seealso:: * :pep:`612` -- Parameter Specification Variables (the PEP which introduced - ``ParamSpec`` and ``Concatenate``). - * :class:`Callable` and :class:`Concatenate`. + ``ParamSpec`` and ``Concatenate``) + * :data:`Concatenate` + * :ref:`annotating-callables` .. data:: ParamSpecArgs .. data:: ParamSpecKwargs @@ -1962,7 +1992,9 @@ without the dedicated syntax, as documented below. Other special directives """""""""""""""""""""""" -These are not used in annotations. They are building blocks for declaring types. +These functions and classes should not be used directly as annotations. +Their intended purpose is to be building blocks for creating and declaring +types. .. class:: NamedTuple @@ -2139,7 +2171,7 @@ These are not used in annotations. They are building blocks for declaring types. methods or attributes, not their type signatures or types. For example, :class:`ssl.SSLObject` is a class, therefore it passes an :func:`issubclass` - check against :data:`Callable`. However, the + check against :ref:`Callable `. However, the ``ssl.SSLObject.__init__`` method exists only to raise a :exc:`TypeError` with a more informative message, therefore making it impossible to call (instantiate) :class:`ssl.SSLObject`. @@ -2383,1093 +2415,1200 @@ These are not used in annotations. They are building blocks for declaring types. .. versionchanged:: 3.11 Added support for generic ``TypedDict``\ s. -Generic concrete collections ----------------------------- - -Corresponding to built-in types -""""""""""""""""""""""""""""""" +Protocols +--------- -.. class:: Dict(dict, MutableMapping[KT, VT]) +The following protocols are provided by the typing module. All are decorated +with :func:`@runtime_checkable `. - Deprecated alias to :class:`dict`. +.. class:: SupportsAbs - Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`Mapping` - rather than to use :class:`dict` or :class:`!typing.Dict`. + An ABC with one abstract method ``__abs__`` that is covariant + in its return type. - This type can be used as follows:: +.. class:: SupportsBytes - def count_words(text: str) -> Dict[str, int]: - ... + An ABC with one abstract method ``__bytes__``. - .. deprecated:: 3.9 - :class:`builtins.dict ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. class:: SupportsComplex -.. class:: List(list, MutableSequence[T]) + An ABC with one abstract method ``__complex__``. - Deprecated alias to :class:`list`. +.. class:: SupportsFloat - Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`Sequence` or - :class:`Iterable` rather than to use :class:`list` or :class:`!typing.List`. + An ABC with one abstract method ``__float__``. - This type may be used as follows:: +.. class:: SupportsIndex - def vec2[T: (int, float)](x: T, y: T) -> List[T]: - return [x, y] + An ABC with one abstract method ``__index__``. - def keep_positives[T: (int, float)](vector: Sequence[T]) -> List[T]: - return [item for item in vector if item > 0] + .. versionadded:: 3.8 - .. deprecated:: 3.9 - :class:`builtins.list ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. class:: SupportsInt -.. class:: Set(set, MutableSet[T]) + An ABC with one abstract method ``__int__``. - Deprecated alias to :class:`builtins.set `. +.. class:: SupportsRound - Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`AbstractSet` - rather than to use :class:`set` or :class:`!typing.Set`. + An ABC with one abstract method ``__round__`` + that is covariant in its return type. - .. deprecated:: 3.9 - :class:`builtins.set ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +ABCs for working with IO +------------------------ -.. class:: FrozenSet(frozenset, AbstractSet[T_co]) +.. class:: IO + TextIO + BinaryIO - Deprecated alias to :class:`builtins.frozenset `. + Generic type ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` + and ``BinaryIO(IO[bytes])`` + represent the types of I/O streams such as returned by + :func:`open`. - .. deprecated:: 3.9 - :class:`builtins.frozenset ` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +Functions and decorators +------------------------ -.. note:: :data:`Tuple` is a special form. +.. function:: cast(typ, val) -Corresponding to types in :mod:`collections` -"""""""""""""""""""""""""""""""""""""""""""" + Cast a value to a type. -.. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) + This returns the value unchanged. To the type checker this + signals that the return value has the designated type, but at + runtime we intentionally don't check anything (we want this + to be as fast as possible). - Deprecated alias to :class:`collections.defaultdict`. +.. function:: assert_type(val, typ, /) - .. versionadded:: 3.5.2 + Ask a static type checker to confirm that *val* has an inferred type of *typ*. - .. deprecated:: 3.9 - :class:`collections.defaultdict` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + At runtime this does nothing: it returns the first argument unchanged with no + checks or side effects, no matter the actual type of the argument. -.. class:: OrderedDict(collections.OrderedDict, MutableMapping[KT, VT]) + When a static type checker encounters a call to ``assert_type()``, it + emits an error if the value is not of the specified type:: - Deprecated alias to :class:`collections.OrderedDict`. + def greet(name: str) -> None: + assert_type(name, str) # OK, inferred type of `name` is `str` + assert_type(name, int) # type checker error - .. versionadded:: 3.7.2 + This function is useful for ensuring the type checker's understanding of a + script is in line with the developer's intentions:: - .. deprecated:: 3.9 - :class:`collections.OrderedDict` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + def complex_function(arg: object): + # Do some complex type-narrowing logic, + # after which we hope the inferred type will be `int` + ... + # Test whether the type checker correctly understands our function + assert_type(arg, int) -.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + .. versionadded:: 3.11 - Deprecated alias to :class:`collections.ChainMap`. +.. function:: assert_never(arg, /) - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 + Ask a static type checker to confirm that a line of code is unreachable. - .. deprecated:: 3.9 - :class:`collections.ChainMap` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Example:: -.. class:: Counter(collections.Counter, Dict[T, int]) + def int_or_str(arg: int | str) -> None: + match arg: + case int(): + print("It's an int") + case str(): + print("It's a str") + case _ as unreachable: + assert_never(unreachable) - Deprecated alias to :class:`collections.Counter`. + Here, the annotations allow the type checker to infer that the + last case can never execute, because ``arg`` is either + an :class:`int` or a :class:`str`, and both options are covered by + earlier cases. - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 + If a type checker finds that a call to ``assert_never()`` is + reachable, it will emit an error. For example, if the type annotation + for ``arg`` was instead ``int | str | float``, the type checker would + emit an error pointing out that ``unreachable`` is of type :class:`float`. + For a call to ``assert_never`` to pass type checking, the inferred type of + the argument passed in must be the bottom type, :data:`Never`, and nothing + else. - .. deprecated:: 3.9 - :class:`collections.Counter` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + At runtime, this throws an exception when called. -.. class:: Deque(deque, MutableSequence[T]) + .. seealso:: + `Unreachable Code and Exhaustiveness Checking + `__ has more + information about exhaustiveness checking with static typing. - Deprecated alias to :class:`collections.deque`. + .. versionadded:: 3.11 - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 +.. function:: reveal_type(obj, /) - .. deprecated:: 3.9 - :class:`collections.deque` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Reveal the inferred static type of an expression. -Other concrete types -"""""""""""""""""""" + When a static type checker encounters a call to this function, + it emits a diagnostic with the type of the argument. For example:: -.. class:: IO - TextIO - BinaryIO + x: int = 1 + reveal_type(x) # Revealed type is "builtins.int" - Generic type ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` - and ``BinaryIO(IO[bytes])`` - represent the types of I/O streams such as returned by - :func:`open`. - - .. deprecated-removed:: 3.8 3.13 - The ``typing.io`` namespace is deprecated and will be removed. - These types should be directly imported from ``typing`` instead. - -.. class:: Pattern - Match + This can be useful when you want to debug how your type checker + handles a particular piece of code. - Deprecated aliases corresponding to the return types from - :func:`re.compile` and :func:`re.match`. + The function returns its argument unchanged, which allows using + it within an expression:: - These types (and the corresponding functions) are generic over - :data:`AnyStr`. ``Pattern`` can be specialised as ``Pattern[str]`` or - ``Pattern[bytes]``; ``Match`` can be specialised as ``Match[str]`` or - ``Match[bytes]``. + x = reveal_type(1) # Revealed type is "builtins.int" - .. deprecated-removed:: 3.8 3.13 - The ``typing.re`` namespace is deprecated and will be removed. - These types should be directly imported from ``typing`` instead. + Most type checkers support ``reveal_type()`` anywhere, even if the + name is not imported from ``typing``. Importing the name from + ``typing`` allows your code to run without runtime errors and + communicates intent more clearly. - .. deprecated:: 3.9 - Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. - See :pep:`585` and :ref:`types-genericalias`. + At runtime, this function prints the runtime type of its argument to stderr + and returns it unchanged:: -.. class:: Text + x = reveal_type(1) # prints "Runtime type is int" + print(x) # prints "1" - Deprecated alias for :class:`str`. + .. versionadded:: 3.11 - ``Text`` is provided to supply a forward - compatible path for Python 2 code: in Python 2, ``Text`` is an alias for - ``unicode``. +.. decorator:: dataclass_transform(*, eq_default=True, order_default=False, \ + kw_only_default=False, frozen_default=False, \ + field_specifiers=(), **kwargs) - Use ``Text`` to indicate that a value must contain a unicode string in - a manner that is compatible with both Python 2 and Python 3:: + Decorator to mark an object as providing + :func:`dataclass `-like behavior. - def add_unicode_checkmark(text: Text) -> Text: - return text + u' \u2713' + ``dataclass_transform`` may be used to + decorate a class, metaclass, or a function that is itself a decorator. + The presence of ``@dataclass_transform()`` tells a static type checker that the + decorated object performs runtime "magic" that + transforms a class in a similar way to + :func:`@dataclasses.dataclass `. - .. versionadded:: 3.5.2 + Example usage with a decorator function: - .. deprecated:: 3.11 - Python 2 is no longer supported, and most type checkers also no longer - support type checking Python 2 code. Removal of the alias is not - currently planned, but users are encouraged to use - :class:`str` instead of ``Text``. + .. testcode:: -Abstract Base Classes ---------------------- + @dataclass_transform() + def create_model[T](cls: type[T]) -> type[T]: + ... + return cls -Corresponding to collections in :mod:`collections.abc` -"""""""""""""""""""""""""""""""""""""""""""""""""""""" + @create_model + class CustomerModel: + id: int + name: str -.. class:: AbstractSet(Collection[T_co]) + On a base class:: - Deprecated alias to :class:`collections.abc.Set`. + @dataclass_transform() + class ModelBase: ... - .. deprecated:: 3.9 - :class:`collections.abc.Set` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + class CustomerModel(ModelBase): + id: int + name: str -.. class:: ByteString(Sequence[int]) + On a metaclass:: - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview` of byte sequences. + @dataclass_transform() + class ModelMeta(type): ... - .. deprecated-removed:: 3.9 3.14 - Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. + class ModelBase(metaclass=ModelMeta): ... -.. class:: Collection(Sized, Iterable[T_co], Container[T_co]) + class CustomerModel(ModelBase): + id: int + name: str - Deprecated alias to :class:`collections.abc.Collection`. + The ``CustomerModel`` classes defined above will + be treated by type checkers similarly to classes created with + :func:`@dataclasses.dataclass `. + For example, type checkers will assume these classes have + ``__init__`` methods that accept ``id`` and ``name``. - .. versionadded:: 3.6.0 + The decorated class, metaclass, or function may accept the following bool + arguments which type checkers will assume have the same effect as they + would have on the + :func:`@dataclasses.dataclass` decorator: ``init``, + ``eq``, ``order``, ``unsafe_hash``, ``frozen``, ``match_args``, + ``kw_only``, and ``slots``. It must be possible for the value of these + arguments (``True`` or ``False``) to be statically evaluated. - .. deprecated:: 3.9 - :class:`collections.abc.Collection` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + The arguments to the ``dataclass_transform`` decorator can be used to + customize the default behaviors of the decorated class, metaclass, or + function: -.. class:: Container(Generic[T_co]) + :param bool eq_default: + Indicates whether the ``eq`` parameter is assumed to be + ``True`` or ``False`` if it is omitted by the caller. + Defaults to ``True``. - Deprecated alias to :class:`collections.abc.Container`. + :param bool order_default: + Indicates whether the ``order`` parameter is + assumed to be ``True`` or ``False`` if it is omitted by the caller. + Defaults to ``False``. - .. deprecated:: 3.9 - :class:`collections.abc.Container` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + :param bool kw_only_default: + Indicates whether the ``kw_only`` parameter is + assumed to be ``True`` or ``False`` if it is omitted by the caller. + Defaults to ``False``. -.. class:: ItemsView(MappingView, AbstractSet[tuple[KT_co, VT_co]]) + :param bool frozen_default: + Indicates whether the ``frozen`` parameter is + assumed to be ``True`` or ``False`` if it is omitted by the caller. + Defaults to ``False``. - Deprecated alias to :class:`collections.abc.ItemsView`. + .. versionadded:: 3.12 - .. deprecated:: 3.9 - :class:`collections.abc.ItemsView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + :param field_specifiers: + Specifies a static list of supported classes + or functions that describe fields, similar to :func:`dataclasses.field`. + Defaults to ``()``. + :type field_specifiers: tuple[Callable[..., Any], ...] -.. class:: KeysView(MappingView, AbstractSet[KT_co]) + :param Any \**kwargs: + Arbitrary other keyword arguments are accepted in order to allow for + possible future extensions. - Deprecated alias to :class:`collections.abc.KeysView`. + Type checkers recognize the following optional parameters on field + specifiers: - .. deprecated:: 3.9 - :class:`collections.abc.KeysView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + .. list-table:: **Recognised parameters for field specifiers** + :header-rows: 1 + :widths: 20 80 + + * - Parameter name + - Description + * - ``init`` + - Indicates whether the field should be included in the + synthesized ``__init__`` method. If unspecified, ``init`` defaults to + ``True``. + * - ``default`` + - Provides the default value for the field. + * - ``default_factory`` + - Provides a runtime callback that returns the + default value for the field. If neither ``default`` nor + ``default_factory`` are specified, the field is assumed to have no + default value and must be provided a value when the class is + instantiated. + * - ``factory`` + - An alias for the ``default_factory`` parameter on field specifiers. + * - ``kw_only`` + - Indicates whether the field should be marked as + keyword-only. If ``True``, the field will be keyword-only. If + ``False``, it will not be keyword-only. If unspecified, the value of + the ``kw_only`` parameter on the object decorated with + ``dataclass_transform`` will be used, or if that is unspecified, the + value of ``kw_only_default`` on ``dataclass_transform`` will be used. + * - ``alias`` + - Provides an alternative name for the field. This alternative + name is used in the synthesized ``__init__`` method. -.. class:: Mapping(Collection[KT], Generic[KT, VT_co]) + At runtime, this decorator records its arguments in the + ``__dataclass_transform__`` attribute on the decorated object. + It has no other runtime effect. - Deprecated alias to :class:`collections.abc.Mapping`. + See :pep:`681` for more details. - This type can be used as follows:: + .. versionadded:: 3.11 - def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: - return word_list[word] +.. decorator:: overload - .. deprecated:: 3.9 - :class:`collections.abc.Mapping` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Decorator for creating overloaded functions and methods. -.. class:: MappingView(Sized) + The ``@overload`` decorator allows describing functions and methods + that support multiple different combinations of argument types. A series + of ``@overload``-decorated definitions must be followed by exactly one + non-``@overload``-decorated definition (for the same function/method). - Deprecated alias to :class:`collections.abc.MappingView`. + ``@overload``-decorated definitions are for the benefit of the + type checker only, since they will be overwritten by the + non-``@overload``-decorated definition. The non-``@overload``-decorated + definition, meanwhile, will be used at + runtime but should be ignored by a type checker. At runtime, calling + an ``@overload``-decorated function directly will raise + :exc:`NotImplementedError`. - .. deprecated:: 3.9 - :class:`collections.abc.MappingView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + An example of overload that gives a more + precise type than can be expressed using a union or a type variable: -.. class:: MutableMapping(Mapping[KT, VT]) + .. testcode:: - Deprecated alias to :class:`collections.abc.MutableMapping`. + @overload + def process(response: None) -> None: + ... + @overload + def process(response: int) -> tuple[int, str]: + ... + @overload + def process(response: bytes) -> str: + ... + def process(response): + ... # actual implementation goes here - .. deprecated:: 3.9 - :class:`collections.abc.MutableMapping` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + See :pep:`484` for more details and comparison with other typing semantics. -.. class:: MutableSequence(Sequence[T]) + .. versionchanged:: 3.11 + Overloaded functions can now be introspected at runtime using + :func:`get_overloads`. - Deprecated alias to :class:`collections.abc.MutableSequence`. - .. deprecated:: 3.9 - :class:`collections.abc.MutableSequence` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. function:: get_overloads(func) -.. class:: MutableSet(AbstractSet[T]) + Return a sequence of :func:`@overload `-decorated definitions for + *func*. - Deprecated alias to :class:`collections.abc.MutableSet`. + *func* is the function object for the implementation of the + overloaded function. For example, given the definition of ``process`` in + the documentation for :func:`@overload `, + ``get_overloads(process)`` will return a sequence of three function objects + for the three defined overloads. If called on a function with no overloads, + ``get_overloads()`` returns an empty sequence. - .. deprecated:: 3.9 - :class:`collections.abc.MutableSet` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + ``get_overloads()`` can be used for introspecting an overloaded function at + runtime. -.. class:: Sequence(Reversible[T_co], Collection[T_co]) + .. versionadded:: 3.11 - Deprecated alias to :class:`collections.abc.Sequence`. - .. deprecated:: 3.9 - :class:`collections.abc.Sequence` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. function:: clear_overloads() -.. class:: ValuesView(MappingView, Collection[_VT_co]) + Clear all registered overloads in the internal registry. - Deprecated alias to :class:`collections.abc.ValuesView`. + This can be used to reclaim the memory used by the registry. - .. deprecated:: 3.9 - :class:`collections.abc.ValuesView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + .. versionadded:: 3.11 -Corresponding to other types in :mod:`collections.abc` -"""""""""""""""""""""""""""""""""""""""""""""""""""""" -.. class:: Iterable(Generic[T_co]) +.. decorator:: final - Deprecated alias to :class:`collections.abc.Iterable`. + Decorator to indicate final methods and final classes. - .. deprecated:: 3.9 - :class:`collections.abc.Iterable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Decorating a method with ``@final`` indicates to a type checker that the + method cannot be overridden in a subclass. Decorating a class with ``@final`` + indicates that it cannot be subclassed. -.. class:: Iterator(Iterable[T_co]) + For example:: - Deprecated alias to :class:`collections.abc.Iterator`. + class Base: + @final + def done(self) -> None: + ... + class Sub(Base): + def done(self) -> None: # Error reported by type checker + ... - .. deprecated:: 3.9 - :class:`collections.abc.Iterator` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + @final + class Leaf: + ... + class Other(Leaf): # Error reported by type checker + ... -.. class:: Generator(Iterator[YieldType], Generic[YieldType, SendType, ReturnType]) + There is no runtime checking of these properties. See :pep:`591` for + more details. - Deprecated alias to :class:`collections.abc.Generator`. + .. versionadded:: 3.8 - A generator can be annotated by the generic type - ``Generator[YieldType, SendType, ReturnType]``. For example:: + .. versionchanged:: 3.11 + The decorator will now attempt to set a ``__final__`` attribute to ``True`` + on the decorated object. Thus, a check like + ``if getattr(obj, "__final__", False)`` can be used at runtime + to determine whether an object ``obj`` has been marked as final. + If the decorated object does not support setting attributes, + the decorator returns the object unchanged without raising an exception. - def echo_round() -> Generator[int, float, str]: - sent = yield 0 - while sent >= 0: - sent = yield round(sent) - return 'Done' - Note that unlike many other generics in the typing module, the ``SendType`` - of :class:`Generator` behaves contravariantly, not covariantly or - invariantly. +.. decorator:: no_type_check - If your generator will only yield values, set the ``SendType`` and - ``ReturnType`` to ``None``:: + Decorator to indicate that annotations are not type hints. - def infinite_stream(start: int) -> Generator[int, None, None]: - while True: - yield start - start += 1 + This works as a class or function :term:`decorator`. With a class, it + applies recursively to all methods and classes defined in that class + (but not to methods defined in its superclasses or subclasses). Type + checkers will ignore all annotations in a function or class with this + decorator. - Alternatively, annotate your generator as having a return type of - either ``Iterable[YieldType]`` or ``Iterator[YieldType]``:: + ``@no_type_check`` mutates the decorated object in place. - def infinite_stream(start: int) -> Iterator[int]: - while True: - yield start - start += 1 +.. decorator:: no_type_check_decorator - .. deprecated:: 3.9 - :class:`collections.abc.Generator` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Decorator to give another decorator the :func:`no_type_check` effect. -.. class:: Hashable + This wraps the decorator with something that wraps the decorated + function in :func:`no_type_check`. - Deprecated alias to :class:`collections.abc.Hashable`. - .. deprecated:: 3.12 - Use :class:`collections.abc.Hashable` directly instead. +.. decorator:: override -.. class:: Reversible(Iterable[T_co]) + Decorator to indicate that a method in a subclass is intended to override a + method or attribute in a superclass. - Deprecated alias to :class:`collections.abc.Reversible`. + Type checkers should emit an error if a method decorated with ``@override`` + does not, in fact, override anything. + This helps prevent bugs that may occur when a base class is changed without + an equivalent change to a child class. - .. deprecated:: 3.9 - :class:`collections.abc.Reversible` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + For example: -.. class:: Sized + .. testcode:: - Deprecated alias to :class:`collections.abc.Sized`. + class Base: + def log_status(self) -> None: + ... - .. deprecated:: 3.12 - Use :class:`collections.abc.Sized` directly instead. + class Sub(Base): + @override + def log_status(self) -> None: # Okay: overrides Base.log_status + ... -Asynchronous programming -"""""""""""""""""""""""" + @override + def done(self) -> None: # Error reported by type checker + ... -.. class:: Coroutine(Awaitable[ReturnType], Generic[YieldType, SendType, ReturnType]) + There is no runtime checking of this property. - Deprecated alias to :class:`collections.abc.Coroutine`. + The decorator will attempt to set an ``__override__`` attribute to ``True`` on + the decorated object. Thus, a check like + ``if getattr(obj, "__override__", False)`` can be used at runtime to determine + whether an object ``obj`` has been marked as an override. If the decorated object + does not support setting attributes, the decorator returns the object unchanged + without raising an exception. - The variance and order of type variables - correspond to those of :class:`Generator`, for example:: + See :pep:`698` for more details. - from collections.abc import Coroutine - c: Coroutine[list[str], str, int] # Some coroutine defined elsewhere - x = c.send('hi') # Inferred type of 'x' is list[str] - async def bar() -> None: - y = await c # Inferred type of 'y' is int + .. versionadded:: 3.12 - .. versionadded:: 3.5.3 - .. deprecated:: 3.9 - :class:`collections.abc.Coroutine` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. decorator:: type_check_only -.. class:: AsyncGenerator(AsyncIterator[YieldType], Generic[YieldType, SendType]) + Decorator to mark a class or function as unavailable at runtime. - Deprecated alias to :class:`collections.abc.AsyncGenerator`. + This decorator is itself not available at runtime. It is mainly + intended to mark classes that are defined in type stub files if + an implementation returns an instance of a private class:: - An async generator can be annotated by the generic type - ``AsyncGenerator[YieldType, SendType]``. For example:: + @type_check_only + class Response: # private or not available at runtime + code: int + def get_header(self, name: str) -> str: ... - async def echo_round() -> AsyncGenerator[int, float]: - sent = yield 0 - while sent >= 0.0: - rounded = await round(sent) - sent = yield rounded + def fetch_response() -> Response: ... - Unlike normal generators, async generators cannot return a value, so there - is no ``ReturnType`` type parameter. As with :class:`Generator`, the - ``SendType`` behaves contravariantly. + Note that returning instances of private classes is not recommended. + It is usually preferable to make such classes public. - If your generator will only yield values, set the ``SendType`` to - ``None``:: +Introspection helpers +--------------------- - async def infinite_stream(start: int) -> AsyncGenerator[int, None]: - while True: - yield start - start = await increment(start) +.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) - Alternatively, annotate your generator as having a return type of - either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: + Return a dictionary containing type hints for a function, method, module + or class object. - async def infinite_stream(start: int) -> AsyncIterator[int]: - while True: - yield start - start = await increment(start) + This is often the same as ``obj.__annotations__``. In addition, + forward references encoded as string literals are handled by evaluating + them in ``globals`` and ``locals`` namespaces. For a class ``C``, return + a dictionary constructed by merging all the ``__annotations__`` along + ``C.__mro__`` in reverse order. - .. versionadded:: 3.6.1 + The function recursively replaces all ``Annotated[T, ...]`` with ``T``, + unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for + more information). For example: - .. deprecated:: 3.9 - :class:`collections.abc.AsyncGenerator` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + .. testcode:: -.. class:: AsyncIterable(Generic[T_co]) + class Student(NamedTuple): + name: Annotated[str, 'some marker'] - Deprecated alias to :class:`collections.abc.AsyncIterable`. + assert get_type_hints(Student) == {'name': str} + assert get_type_hints(Student, include_extras=False) == {'name': str} + assert get_type_hints(Student, include_extras=True) == { + 'name': Annotated[str, 'some marker'] + } - .. versionadded:: 3.5.2 + .. note:: - .. deprecated:: 3.9 - :class:`collections.abc.AsyncIterable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + :func:`get_type_hints` does not work with imported + :ref:`type aliases ` that include forward references. + Enabling postponed evaluation of annotations (:pep:`563`) may remove + the need for most forward references. -.. class:: AsyncIterator(AsyncIterable[T_co]) + .. versionchanged:: 3.9 + Added ``include_extras`` parameter as part of :pep:`593`. + See the documentation on :data:`Annotated` for more information. - Deprecated alias to :class:`collections.abc.AsyncIterator`. + .. versionchanged:: 3.11 + Previously, ``Optional[t]`` was added for function and method annotations + if a default value equal to ``None`` was set. + Now the annotation is returned unchanged. - .. versionadded:: 3.5.2 +.. function:: get_origin(tp) - .. deprecated:: 3.9 - :class:`collections.abc.AsyncIterator` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Get the unsubscripted version of a type: for a typing object of the form + ``X[Y, Z, ...]`` return ``X``. -.. class:: Awaitable(Generic[T_co]) + If ``X`` is a typing-module alias for a builtin or + :mod:`collections` class, it will be normalized to the original class. + If ``X`` is an instance of :class:`ParamSpecArgs` or :class:`ParamSpecKwargs`, + return the underlying :class:`ParamSpec`. + Return ``None`` for unsupported objects. - Deprecated alias to :class:`collections.abc.Awaitable`. + Examples: + + .. testcode:: + + assert get_origin(str) is None + assert get_origin(Dict[str, int]) is dict + assert get_origin(Union[int, str]) is Union + P = ParamSpec('P') + assert get_origin(P.args) is P + assert get_origin(P.kwargs) is P + + .. versionadded:: 3.8 + +.. function:: get_args(tp) + + Get type arguments with all substitutions performed: for a typing object + of the form ``X[Y, Z, ...]`` return ``(Y, Z, ...)``. + + If ``X`` is a union or :class:`Literal` contained in another + generic type, the order of ``(Y, Z, ...)`` may be different from the order + of the original arguments ``[Y, Z, ...]`` due to type caching. + Return ``()`` for unsupported objects. + + Examples: + + .. testcode:: + + assert get_args(int) == () + assert get_args(Dict[int, str]) == (int, str) + assert get_args(Union[int, str]) == (int, str) + + .. versionadded:: 3.8 + +.. function:: is_typeddict(tp) + + Check if a type is a :class:`TypedDict`. + + For example: + + .. testcode:: + + class Film(TypedDict): + title: str + year: int + + assert is_typeddict(Film) + assert not is_typeddict(list | str) + + # TypedDict is a factory for creating typed dicts, + # not a typed dict itself + assert not is_typeddict(TypedDict) + + .. versionadded:: 3.10 + +.. class:: ForwardRef + + Class used for internal typing representation of string forward references. + + For example, ``List["SomeClass"]`` is implicitly transformed into + ``List[ForwardRef("SomeClass")]``. ``ForwardRef`` should not be instantiated by + a user, but may be used by introspection tools. + + .. note:: + :pep:`585` generic types such as ``list["SomeClass"]`` will not be + implicitly transformed into ``list[ForwardRef("SomeClass")]`` and thus + will not automatically resolve to ``list[SomeClass]``. + + .. versionadded:: 3.7.4 + +Constant +-------- + +.. data:: TYPE_CHECKING + + A special constant that is assumed to be ``True`` by 3rd party static + type checkers. It is ``False`` at runtime. + + Usage:: + + if TYPE_CHECKING: + import expensive_mod + + def fun(arg: 'expensive_mod.SomeType') -> None: + local_var: expensive_mod.AnotherType = other_fun() + + The first type annotation must be enclosed in quotes, making it a + "forward reference", to hide the ``expensive_mod`` reference from the + interpreter runtime. Type annotations for local variables are not + evaluated, so the second annotation does not need to be enclosed in quotes. + + .. note:: + + If ``from __future__ import annotations`` is used, + annotations are not evaluated at function definition time. + Instead, they are stored as strings in ``__annotations__``. + This makes it unnecessary to use quotes around the annotation + (see :pep:`563`). .. versionadded:: 3.5.2 - .. deprecated:: 3.9 - :class:`collections.abc.Awaitable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. _generic-concrete-collections: +.. _deprecated-aliases: +Deprecated aliases +------------------ -Context manager types -""""""""""""""""""""" +This module defines several deprecated aliases to pre-existing +standard library classes. These were originally included in the typing +module in order to support parameterizing these generic classes using ``[]``. +However, the aliases became redundant in Python 3.9 when the +corresponding pre-existing classes were enhanced to support ``[]`` (see +:pep:`585`). -.. class:: ContextManager(Generic[T_co]) +The redundant types are deprecated as of Python 3.9. However, while the aliases +may be removed at some point, removal of these aliases is not currently +planned. As such, no deprecation warnings are currently issued by the +interpreter for these aliases. - Deprecated alias to :class:`contextlib.AbstractContextManager`. +If at some point it is decided to remove these deprecated aliases, a +deprecation warning will be issued by the interpreter for at least two releases +prior to removal. The aliases are guaranteed to remain in the typing module +without deprecation warnings until at least Python 3.14. - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.0 +Type checkers are encouraged to flag uses of the deprecated types if the +program they are checking targets a minimum Python version of 3.9 or newer. - .. deprecated:: 3.9 - :class:`contextlib.AbstractContextManager` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. _corresponding-to-built-in-types: -.. class:: AsyncContextManager(Generic[T_co]) +Aliases to built-in types +""""""""""""""""""""""""" - Deprecated alias to :class:`contextlib.AbstractAsyncContextManager`. +.. class:: Dict(dict, MutableMapping[KT, VT]) - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.2 + Deprecated alias to :class:`dict`. - .. deprecated:: 3.9 - :class:`contextlib.AbstractAsyncContextManager` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Note that to annotate arguments, it is preferred + to use an abstract collection type such as :class:`Mapping` + rather than to use :class:`dict` or :class:`!typing.Dict`. -Protocols ---------- + This type can be used as follows:: -These protocols are decorated with :func:`runtime_checkable`. + def count_words(text: str) -> Dict[str, int]: + ... -.. class:: SupportsAbs + .. deprecated:: 3.9 + :class:`builtins.dict ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - An ABC with one abstract method ``__abs__`` that is covariant - in its return type. +.. class:: List(list, MutableSequence[T]) -.. class:: SupportsBytes + Deprecated alias to :class:`list`. - An ABC with one abstract method ``__bytes__``. + Note that to annotate arguments, it is preferred + to use an abstract collection type such as :class:`Sequence` or + :class:`Iterable` rather than to use :class:`list` or :class:`!typing.List`. -.. class:: SupportsComplex + This type may be used as follows:: - An ABC with one abstract method ``__complex__``. + def vec2[T: (int, float)](x: T, y: T) -> List[T]: + return [x, y] -.. class:: SupportsFloat + def keep_positives[T: (int, float)](vector: Sequence[T]) -> List[T]: + return [item for item in vector if item > 0] - An ABC with one abstract method ``__float__``. + .. deprecated:: 3.9 + :class:`builtins.list ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -.. class:: SupportsIndex +.. class:: Set(set, MutableSet[T]) - An ABC with one abstract method ``__index__``. + Deprecated alias to :class:`builtins.set `. - .. versionadded:: 3.8 + Note that to annotate arguments, it is preferred + to use an abstract collection type such as :class:`AbstractSet` + rather than to use :class:`set` or :class:`!typing.Set`. -.. class:: SupportsInt + .. deprecated:: 3.9 + :class:`builtins.set ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - An ABC with one abstract method ``__int__``. +.. class:: FrozenSet(frozenset, AbstractSet[T_co]) -.. class:: SupportsRound + Deprecated alias to :class:`builtins.frozenset `. - An ABC with one abstract method ``__round__`` - that is covariant in its return type. + .. deprecated:: 3.9 + :class:`builtins.frozenset ` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -Functions and decorators ------------------------- +.. data:: Tuple -.. function:: cast(typ, val) + Deprecated alias for :class:`tuple`. - Cast a value to a type. + :class:`tuple` and ``Tuple`` are special-cased in the type system; see + :ref:`annotating-tuples` for more details. - This returns the value unchanged. To the type checker this - signals that the return value has the designated type, but at - runtime we intentionally don't check anything (we want this - to be as fast as possible). + .. deprecated:: 3.9 + :class:`builtins.tuple ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -.. function:: assert_type(val, typ, /) +.. class:: Type(Generic[CT_co]) - Ask a static type checker to confirm that *val* has an inferred type of *typ*. + Deprecated alias to :class:`type`. - At runtime this does nothing: it returns the first argument unchanged with no - checks or side effects, no matter the actual type of the argument. + See :ref:`type-of-class-objects` for details on using :class:`type` or + ``typing.Type`` in type annotations. - When a static type checker encounters a call to ``assert_type()``, it - emits an error if the value is not of the specified type:: + .. versionadded:: 3.5.2 - def greet(name: str) -> None: - assert_type(name, str) # OK, inferred type of `name` is `str` - assert_type(name, int) # type checker error + .. deprecated:: 3.9 + :class:`builtins.type ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - This function is useful for ensuring the type checker's understanding of a - script is in line with the developer's intentions:: +.. _corresponding-to-types-in-collections: - def complex_function(arg: object): - # Do some complex type-narrowing logic, - # after which we hope the inferred type will be `int` - ... - # Test whether the type checker correctly understands our function - assert_type(arg, int) +Aliases to types in :mod:`collections` +"""""""""""""""""""""""""""""""""""""" - .. versionadded:: 3.11 +.. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) -.. function:: assert_never(arg, /) + Deprecated alias to :class:`collections.defaultdict`. - Ask a static type checker to confirm that a line of code is unreachable. + .. versionadded:: 3.5.2 - Example:: + .. deprecated:: 3.9 + :class:`collections.defaultdict` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - def int_or_str(arg: int | str) -> None: - match arg: - case int(): - print("It's an int") - case str(): - print("It's a str") - case _ as unreachable: - assert_never(unreachable) +.. class:: OrderedDict(collections.OrderedDict, MutableMapping[KT, VT]) - Here, the annotations allow the type checker to infer that the - last case can never execute, because ``arg`` is either - an :class:`int` or a :class:`str`, and both options are covered by - earlier cases. + Deprecated alias to :class:`collections.OrderedDict`. - If a type checker finds that a call to ``assert_never()`` is - reachable, it will emit an error. For example, if the type annotation - for ``arg`` was instead ``int | str | float``, the type checker would - emit an error pointing out that ``unreachable`` is of type :class:`float`. - For a call to ``assert_never`` to pass type checking, the inferred type of - the argument passed in must be the bottom type, :data:`Never`, and nothing - else. + .. versionadded:: 3.7.2 - At runtime, this throws an exception when called. + .. deprecated:: 3.9 + :class:`collections.OrderedDict` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. seealso:: - `Unreachable Code and Exhaustiveness Checking - `__ has more - information about exhaustiveness checking with static typing. +.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) - .. versionadded:: 3.11 + Deprecated alias to :class:`collections.ChainMap`. -.. function:: reveal_type(obj, /) + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 - Reveal the inferred static type of an expression. + .. deprecated:: 3.9 + :class:`collections.ChainMap` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - When a static type checker encounters a call to this function, - it emits a diagnostic with the type of the argument. For example:: +.. class:: Counter(collections.Counter, Dict[T, int]) - x: int = 1 - reveal_type(x) # Revealed type is "builtins.int" + Deprecated alias to :class:`collections.Counter`. - This can be useful when you want to debug how your type checker - handles a particular piece of code. + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 - The function returns its argument unchanged, which allows using - it within an expression:: + .. deprecated:: 3.9 + :class:`collections.Counter` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - x = reveal_type(1) # Revealed type is "builtins.int" +.. class:: Deque(deque, MutableSequence[T]) - Most type checkers support ``reveal_type()`` anywhere, even if the - name is not imported from ``typing``. Importing the name from - ``typing`` allows your code to run without runtime errors and - communicates intent more clearly. + Deprecated alias to :class:`collections.deque`. - At runtime, this function prints the runtime type of its argument to stderr - and returns it unchanged:: + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 - x = reveal_type(1) # prints "Runtime type is int" - print(x) # prints "1" + .. deprecated:: 3.9 + :class:`collections.deque` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. versionadded:: 3.11 +.. _other-concrete-types: -.. decorator:: dataclass_transform +Aliases to other concrete types +""""""""""""""""""""""""""""""" - Decorator to mark an object as providing - :func:`~dataclasses.dataclass`-like behavior. + .. deprecated-removed:: 3.8 3.13 + The ``typing.io`` namespace is deprecated and will be removed. + These types should be directly imported from ``typing`` instead. - ``dataclass_transform`` may be used to - decorate a class, metaclass, or a function that is itself a decorator. - The presence of ``@dataclass_transform()`` tells a static type checker that the - decorated object performs runtime "magic" that - transforms a class in a similar way to :func:`dataclasses.dataclass`. +.. class:: Pattern + Match + + Deprecated aliases corresponding to the return types from + :func:`re.compile` and :func:`re.match`. - Example usage with a decorator function: + These types (and the corresponding functions) are generic over + :data:`AnyStr`. ``Pattern`` can be specialised as ``Pattern[str]`` or + ``Pattern[bytes]``; ``Match`` can be specialised as ``Match[str]`` or + ``Match[bytes]``. - .. testcode:: + .. deprecated-removed:: 3.8 3.13 + The ``typing.re`` namespace is deprecated and will be removed. + These types should be directly imported from ``typing`` instead. - @dataclass_transform() - def create_model[T](cls: type[T]) -> type[T]: - ... - return cls + .. deprecated:: 3.9 + Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. + See :pep:`585` and :ref:`types-genericalias`. - @create_model - class CustomerModel: - id: int - name: str +.. class:: Text - On a base class:: + Deprecated alias for :class:`str`. - @dataclass_transform() - class ModelBase: ... + ``Text`` is provided to supply a forward + compatible path for Python 2 code: in Python 2, ``Text`` is an alias for + ``unicode``. - class CustomerModel(ModelBase): - id: int - name: str + Use ``Text`` to indicate that a value must contain a unicode string in + a manner that is compatible with both Python 2 and Python 3:: - On a metaclass:: + def add_unicode_checkmark(text: Text) -> Text: + return text + u' \u2713' - @dataclass_transform() - class ModelMeta(type): ... + .. versionadded:: 3.5.2 - class ModelBase(metaclass=ModelMeta): ... + .. deprecated:: 3.11 + Python 2 is no longer supported, and most type checkers also no longer + support type checking Python 2 code. Removal of the alias is not + currently planned, but users are encouraged to use + :class:`str` instead of ``Text``. - class CustomerModel(ModelBase): - id: int - name: str +.. _abstract-base-classes: +.. _corresponding-to-collections-in-collections-abc: - The ``CustomerModel`` classes defined above will - be treated by type checkers similarly to classes created with - :func:`@dataclasses.dataclass `. - For example, type checkers will assume these classes have - ``__init__`` methods that accept ``id`` and ``name``. +Aliases to container ABCs in :mod:`collections.abc` +""""""""""""""""""""""""""""""""""""""""""""""""""" - The decorated class, metaclass, or function may accept the following bool - arguments which type checkers will assume have the same effect as they - would have on the - :func:`@dataclasses.dataclass` decorator: ``init``, - ``eq``, ``order``, ``unsafe_hash``, ``frozen``, ``match_args``, - ``kw_only``, and ``slots``. It must be possible for the value of these - arguments (``True`` or ``False``) to be statically evaluated. +.. class:: AbstractSet(Collection[T_co]) - The arguments to the ``dataclass_transform`` decorator can be used to - customize the default behaviors of the decorated class, metaclass, or - function: + Deprecated alias to :class:`collections.abc.Set`. - * ``eq_default`` indicates whether the ``eq`` parameter is assumed to be - ``True`` or ``False`` if it is omitted by the caller. - * ``order_default`` indicates whether the ``order`` parameter is - assumed to be True or False if it is omitted by the caller. - * ``kw_only_default`` indicates whether the ``kw_only`` parameter is - assumed to be True or False if it is omitted by the caller. - * ``frozen_default`` indicates whether the ``frozen`` parameter is - assumed to be True or False if it is omitted by the caller. - - .. versionadded:: 3.12 - * ``field_specifiers`` specifies a static list of supported classes - or functions that describe fields, similar to ``dataclasses.field()``. - * Arbitrary other keyword arguments are accepted in order to allow for - possible future extensions. - - Type checkers recognize the following optional arguments on field - specifiers: + .. deprecated:: 3.9 + :class:`collections.abc.Set` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - * ``init`` indicates whether the field should be included in the - synthesized ``__init__`` method. If unspecified, ``init`` defaults to - ``True``. - * ``default`` provides the default value for the field. - * ``default_factory`` provides a runtime callback that returns the - default value for the field. If neither ``default`` nor - ``default_factory`` are specified, the field is assumed to have no - default value and must be provided a value when the class is - instantiated. - * ``factory`` is an alias for ``default_factory``. - * ``kw_only`` indicates whether the field should be marked as - keyword-only. If ``True``, the field will be keyword-only. If - ``False``, it will not be keyword-only. If unspecified, the value of - the ``kw_only`` parameter on the object decorated with - ``dataclass_transform`` will be used, or if that is unspecified, the - value of ``kw_only_default`` on ``dataclass_transform`` will be used. - * ``alias`` provides an alternative name for the field. This alternative - name is used in the synthesized ``__init__`` method. +.. class:: ByteString(Sequence[int]) - At runtime, this decorator records its arguments in the - ``__dataclass_transform__`` attribute on the decorated object. - It has no other runtime effect. + This type represents the types :class:`bytes`, :class:`bytearray`, + and :class:`memoryview` of byte sequences. - See :pep:`681` for more details. + .. deprecated-removed:: 3.9 3.14 + Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. - .. versionadded:: 3.11 +.. class:: Collection(Sized, Iterable[T_co], Container[T_co]) -.. decorator:: overload + Deprecated alias to :class:`collections.abc.Collection`. - Decorator for creating overloaded functions and methods. + .. versionadded:: 3.6.0 - The ``@overload`` decorator allows describing functions and methods - that support multiple different combinations of argument types. A series - of ``@overload``-decorated definitions must be followed by exactly one - non-``@overload``-decorated definition (for the same function/method). + .. deprecated:: 3.9 + :class:`collections.abc.Collection` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - ``@overload``-decorated definitions are for the benefit of the - type checker only, since they will be overwritten by the - non-``@overload``-decorated definition. The non-``@overload``-decorated - definition, meanwhile, will be used at - runtime but should be ignored by a type checker. At runtime, calling - an ``@overload``-decorated function directly will raise - :exc:`NotImplementedError`. +.. class:: Container(Generic[T_co]) - An example of overload that gives a more - precise type than can be expressed using a union or a type variable: + Deprecated alias to :class:`collections.abc.Container`. - .. testcode:: + .. deprecated:: 3.9 + :class:`collections.abc.Container` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - @overload - def process(response: None) -> None: - ... - @overload - def process(response: int) -> tuple[int, str]: - ... - @overload - def process(response: bytes) -> str: - ... - def process(response): - ... # actual implementation goes here +.. class:: ItemsView(MappingView, AbstractSet[tuple[KT_co, VT_co]]) - See :pep:`484` for more details and comparison with other typing semantics. + Deprecated alias to :class:`collections.abc.ItemsView`. - .. versionchanged:: 3.11 - Overloaded functions can now be introspected at runtime using - :func:`get_overloads`. + .. deprecated:: 3.9 + :class:`collections.abc.ItemsView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. +.. class:: KeysView(MappingView, AbstractSet[KT_co]) -.. function:: get_overloads(func) + Deprecated alias to :class:`collections.abc.KeysView`. - Return a sequence of :func:`@overload `-decorated definitions for - *func*. + .. deprecated:: 3.9 + :class:`collections.abc.KeysView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - *func* is the function object for the implementation of the - overloaded function. For example, given the definition of ``process`` in - the documentation for :func:`@overload `, - ``get_overloads(process)`` will return a sequence of three function objects - for the three defined overloads. If called on a function with no overloads, - ``get_overloads()`` returns an empty sequence. +.. class:: Mapping(Collection[KT], Generic[KT, VT_co]) - ``get_overloads()`` can be used for introspecting an overloaded function at - runtime. + Deprecated alias to :class:`collections.abc.Mapping`. - .. versionadded:: 3.11 + This type can be used as follows:: + def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: + return word_list[word] -.. function:: clear_overloads() + .. deprecated:: 3.9 + :class:`collections.abc.Mapping` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - Clear all registered overloads in the internal registry. +.. class:: MappingView(Sized) - This can be used to reclaim the memory used by the registry. + Deprecated alias to :class:`collections.abc.MappingView`. - .. versionadded:: 3.11 + .. deprecated:: 3.9 + :class:`collections.abc.MappingView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. +.. class:: MutableMapping(Mapping[KT, VT]) -.. decorator:: final + Deprecated alias to :class:`collections.abc.MutableMapping`. - Decorator to indicate final methods and final classes. + .. deprecated:: 3.9 + :class:`collections.abc.MutableMapping` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - Decorating a method with ``@final`` indicates to a type checker that the - method cannot be overridden in a subclass. Decorating a class with ``@final`` - indicates that it cannot be subclassed. +.. class:: MutableSequence(Sequence[T]) - For example:: + Deprecated alias to :class:`collections.abc.MutableSequence`. - class Base: - @final - def done(self) -> None: - ... - class Sub(Base): - def done(self) -> None: # Error reported by type checker - ... + .. deprecated:: 3.9 + :class:`collections.abc.MutableSequence` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - @final - class Leaf: - ... - class Other(Leaf): # Error reported by type checker - ... +.. class:: MutableSet(AbstractSet[T]) - There is no runtime checking of these properties. See :pep:`591` for - more details. + Deprecated alias to :class:`collections.abc.MutableSet`. - .. versionadded:: 3.8 + .. deprecated:: 3.9 + :class:`collections.abc.MutableSet` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. versionchanged:: 3.11 - The decorator will now attempt to set a ``__final__`` attribute to ``True`` - on the decorated object. Thus, a check like - ``if getattr(obj, "__final__", False)`` can be used at runtime - to determine whether an object ``obj`` has been marked as final. - If the decorated object does not support setting attributes, - the decorator returns the object unchanged without raising an exception. +.. class:: Sequence(Reversible[T_co], Collection[T_co]) + Deprecated alias to :class:`collections.abc.Sequence`. -.. decorator:: no_type_check + .. deprecated:: 3.9 + :class:`collections.abc.Sequence` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - Decorator to indicate that annotations are not type hints. +.. class:: ValuesView(MappingView, Collection[_VT_co]) - This works as a class or function :term:`decorator`. With a class, it - applies recursively to all methods and classes defined in that class - (but not to methods defined in its superclasses or subclasses). Type - checkers will ignore all annotations in a function or class with this - decorator. + Deprecated alias to :class:`collections.abc.ValuesView`. - ``@no_type_check`` mutates the decorated object in place. + .. deprecated:: 3.9 + :class:`collections.abc.ValuesView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -.. decorator:: no_type_check_decorator +.. _asynchronous-programming: - Decorator to give another decorator the :func:`no_type_check` effect. +Aliases to asynchronous ABCs in :mod:`collections.abc` +"""""""""""""""""""""""""""""""""""""""""""""""""""""" - This wraps the decorator with something that wraps the decorated - function in :func:`no_type_check`. +.. class:: Coroutine(Awaitable[ReturnType], Generic[YieldType, SendType, ReturnType]) + Deprecated alias to :class:`collections.abc.Coroutine`. -.. decorator:: override + The variance and order of type variables + correspond to those of :class:`Generator`, for example:: - Decorator to indicate that a method in a subclass is intended to override a - method or attribute in a superclass. + from collections.abc import Coroutine + c: Coroutine[list[str], str, int] # Some coroutine defined elsewhere + x = c.send('hi') # Inferred type of 'x' is list[str] + async def bar() -> None: + y = await c # Inferred type of 'y' is int - Type checkers should emit an error if a method decorated with ``@override`` - does not, in fact, override anything. - This helps prevent bugs that may occur when a base class is changed without - an equivalent change to a child class. + .. versionadded:: 3.5.3 - For example: + .. deprecated:: 3.9 + :class:`collections.abc.Coroutine` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. testcode:: +.. class:: AsyncGenerator(AsyncIterator[YieldType], Generic[YieldType, SendType]) - class Base: - def log_status(self) -> None: - ... + Deprecated alias to :class:`collections.abc.AsyncGenerator`. - class Sub(Base): - @override - def log_status(self) -> None: # Okay: overrides Base.log_status - ... + An async generator can be annotated by the generic type + ``AsyncGenerator[YieldType, SendType]``. For example:: - @override - def done(self) -> None: # Error reported by type checker - ... + async def echo_round() -> AsyncGenerator[int, float]: + sent = yield 0 + while sent >= 0.0: + rounded = await round(sent) + sent = yield rounded - There is no runtime checking of this property. + Unlike normal generators, async generators cannot return a value, so there + is no ``ReturnType`` type parameter. As with :class:`Generator`, the + ``SendType`` behaves contravariantly. - The decorator will attempt to set an ``__override__`` attribute to ``True`` on - the decorated object. Thus, a check like - ``if getattr(obj, "__override__", False)`` can be used at runtime to determine - whether an object ``obj`` has been marked as an override. If the decorated object - does not support setting attributes, the decorator returns the object unchanged - without raising an exception. + If your generator will only yield values, set the ``SendType`` to + ``None``:: - See :pep:`698` for more details. + async def infinite_stream(start: int) -> AsyncGenerator[int, None]: + while True: + yield start + start = await increment(start) - .. versionadded:: 3.12 + Alternatively, annotate your generator as having a return type of + either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: + async def infinite_stream(start: int) -> AsyncIterator[int]: + while True: + yield start + start = await increment(start) -.. decorator:: type_check_only + .. versionadded:: 3.6.1 - Decorator to mark a class or function as unavailable at runtime. + .. deprecated:: 3.9 + :class:`collections.abc.AsyncGenerator` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - This decorator is itself not available at runtime. It is mainly - intended to mark classes that are defined in type stub files if - an implementation returns an instance of a private class:: +.. class:: AsyncIterable(Generic[T_co]) - @type_check_only - class Response: # private or not available at runtime - code: int - def get_header(self, name: str) -> str: ... + Deprecated alias to :class:`collections.abc.AsyncIterable`. - def fetch_response() -> Response: ... + .. versionadded:: 3.5.2 - Note that returning instances of private classes is not recommended. - It is usually preferable to make such classes public. + .. deprecated:: 3.9 + :class:`collections.abc.AsyncIterable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -Introspection helpers ---------------------- +.. class:: AsyncIterator(AsyncIterable[T_co]) -.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) + Deprecated alias to :class:`collections.abc.AsyncIterator`. - Return a dictionary containing type hints for a function, method, module - or class object. + .. versionadded:: 3.5.2 - This is often the same as ``obj.__annotations__``. In addition, - forward references encoded as string literals are handled by evaluating - them in ``globals`` and ``locals`` namespaces. For a class ``C``, return - a dictionary constructed by merging all the ``__annotations__`` along - ``C.__mro__`` in reverse order. + .. deprecated:: 3.9 + :class:`collections.abc.AsyncIterator` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - The function recursively replaces all ``Annotated[T, ...]`` with ``T``, - unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for - more information). For example: +.. class:: Awaitable(Generic[T_co]) - .. testcode:: + Deprecated alias to :class:`collections.abc.Awaitable`. - class Student(NamedTuple): - name: Annotated[str, 'some marker'] + .. versionadded:: 3.5.2 - assert get_type_hints(Student) == {'name': str} - assert get_type_hints(Student, include_extras=False) == {'name': str} - assert get_type_hints(Student, include_extras=True) == { - 'name': Annotated[str, 'some marker'] - } + .. deprecated:: 3.9 + :class:`collections.abc.Awaitable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. note:: +.. _corresponding-to-other-types-in-collections-abc: - :func:`get_type_hints` does not work with imported - :ref:`type aliases ` that include forward references. - Enabling postponed evaluation of annotations (:pep:`563`) may remove - the need for most forward references. +Aliases to other ABCs in :mod:`collections.abc` +""""""""""""""""""""""""""""""""""""""""""""""" - .. versionchanged:: 3.9 - Added ``include_extras`` parameter as part of :pep:`593`. - See the documentation on :data:`Annotated` for more information. +.. class:: Iterable(Generic[T_co]) - .. versionchanged:: 3.11 - Previously, ``Optional[t]`` was added for function and method annotations - if a default value equal to ``None`` was set. - Now the annotation is returned unchanged. + Deprecated alias to :class:`collections.abc.Iterable`. -.. function:: get_origin(tp) + .. deprecated:: 3.9 + :class:`collections.abc.Iterable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - Get the unsubscripted version of a type: for a typing object of the form - ``X[Y, Z, ...]`` return ``X``. +.. class:: Iterator(Iterable[T_co]) - If ``X`` is a typing-module alias for a builtin or - :mod:`collections` class, it will be normalized to the original class. - If ``X`` is an instance of :class:`ParamSpecArgs` or :class:`ParamSpecKwargs`, - return the underlying :class:`ParamSpec`. - Return ``None`` for unsupported objects. + Deprecated alias to :class:`collections.abc.Iterator`. - Examples: + .. deprecated:: 3.9 + :class:`collections.abc.Iterator` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. testcode:: +.. data:: Callable - assert get_origin(str) is None - assert get_origin(Dict[str, int]) is dict - assert get_origin(Union[int, str]) is Union - P = ParamSpec('P') - assert get_origin(P.args) is P - assert get_origin(P.kwargs) is P + Deprecated alias to :class:`collections.abc.Callable`. - .. versionadded:: 3.8 + See :ref:`annotating-callables` for details on how to use + :class:`collections.abc.Callable` and ``typing.Callable`` in type annotations. -.. function:: get_args(tp) + .. deprecated:: 3.9 + :class:`collections.abc.Callable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - Get type arguments with all substitutions performed: for a typing object - of the form ``X[Y, Z, ...]`` return ``(Y, Z, ...)``. + .. versionchanged:: 3.10 + ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. + See :pep:`612` for more details. - If ``X`` is a union or :class:`Literal` contained in another - generic type, the order of ``(Y, Z, ...)`` may be different from the order - of the original arguments ``[Y, Z, ...]`` due to type caching. - Return ``()`` for unsupported objects. +.. class:: Generator(Iterator[YieldType], Generic[YieldType, SendType, ReturnType]) - Examples: + Deprecated alias to :class:`collections.abc.Generator`. - .. testcode:: + A generator can be annotated by the generic type + ``Generator[YieldType, SendType, ReturnType]``. For example:: - assert get_args(int) == () - assert get_args(Dict[int, str]) == (int, str) - assert get_args(Union[int, str]) == (int, str) + def echo_round() -> Generator[int, float, str]: + sent = yield 0 + while sent >= 0: + sent = yield round(sent) + return 'Done' - .. versionadded:: 3.8 + Note that unlike many other generics in the typing module, the ``SendType`` + of :class:`Generator` behaves contravariantly, not covariantly or + invariantly. -.. function:: is_typeddict(tp) + If your generator will only yield values, set the ``SendType`` and + ``ReturnType`` to ``None``:: - Check if a type is a :class:`TypedDict`. + def infinite_stream(start: int) -> Generator[int, None, None]: + while True: + yield start + start += 1 - For example: + Alternatively, annotate your generator as having a return type of + either ``Iterable[YieldType]`` or ``Iterator[YieldType]``:: - .. testcode:: + def infinite_stream(start: int) -> Iterator[int]: + while True: + yield start + start += 1 - class Film(TypedDict): - title: str - year: int + .. deprecated:: 3.9 + :class:`collections.abc.Generator` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - assert is_typeddict(Film) - assert not is_typeddict(list | str) +.. class:: Hashable - # TypedDict is a factory for creating typed dicts, - # not a typed dict itself - assert not is_typeddict(TypedDict) + Deprecated alias to :class:`collections.abc.Hashable`. - .. versionadded:: 3.10 + .. deprecated:: 3.12 + Use :class:`collections.abc.Hashable` directly instead. -.. class:: ForwardRef +.. class:: Reversible(Iterable[T_co]) - Class used for internal typing representation of string forward references. + Deprecated alias to :class:`collections.abc.Reversible`. - For example, ``List["SomeClass"]`` is implicitly transformed into - ``List[ForwardRef("SomeClass")]``. ``ForwardRef`` should not be instantiated by - a user, but may be used by introspection tools. + .. deprecated:: 3.9 + :class:`collections.abc.Reversible` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. note:: - :pep:`585` generic types such as ``list["SomeClass"]`` will not be - implicitly transformed into ``list[ForwardRef("SomeClass")]`` and thus - will not automatically resolve to ``list[SomeClass]``. +.. class:: Sized - .. versionadded:: 3.7.4 + Deprecated alias to :class:`collections.abc.Sized`. -Constant --------- + .. deprecated:: 3.12 + Use :class:`collections.abc.Sized` directly instead. -.. data:: TYPE_CHECKING +.. _context-manager-types: - A special constant that is assumed to be ``True`` by 3rd party static - type checkers. It is ``False`` at runtime. +Aliases to :mod:`contextlib` ABCs +""""""""""""""""""""""""""""""""" - Usage:: +.. class:: ContextManager(Generic[T_co]) - if TYPE_CHECKING: - import expensive_mod + Deprecated alias to :class:`contextlib.AbstractContextManager`. - def fun(arg: 'expensive_mod.SomeType') -> None: - local_var: expensive_mod.AnotherType = other_fun() + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.0 - The first type annotation must be enclosed in quotes, making it a - "forward reference", to hide the ``expensive_mod`` reference from the - interpreter runtime. Type annotations for local variables are not - evaluated, so the second annotation does not need to be enclosed in quotes. + .. deprecated:: 3.9 + :class:`contextlib.AbstractContextManager` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. note:: +.. class:: AsyncContextManager(Generic[T_co]) - If ``from __future__ import annotations`` is used, - annotations are not evaluated at function definition time. - Instead, they are stored as strings in ``__annotations__``. - This makes it unnecessary to use quotes around the annotation - (see :pep:`563`). + Deprecated alias to :class:`contextlib.AbstractAsyncContextManager`. - .. versionadded:: 3.5.2 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 + + .. deprecated:: 3.9 + :class:`contextlib.AbstractAsyncContextManager` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. Deprecation Timeline of Major Features ====================================== @@ -3478,21 +3617,34 @@ Certain features in ``typing`` are deprecated and may be removed in a future version of Python. The following table summarizes major deprecations for your convenience. This is subject to change, and not all deprecations are listed. -+----------------------------------+---------------+-------------------+----------------+ -| Feature | Deprecated in | Projected removal | PEP/issue | -+==================================+===============+===================+================+ -| ``typing.io`` and ``typing.re`` | 3.8 | 3.13 | :issue:`38291` | -| submodules | | | | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | -| collections | | | | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.ByteString`` | 3.9 | 3.14 | :gh:`91896` | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.Text`` | 3.11 | Undecided | :gh:`92332` | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.Hashable`` and | 3.12 | Undecided | :gh:`94309` | -| ``typing.Sized`` | | | | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.TypeAlias`` | 3.12 | Undecided | :pep:`695` | -+----------------------------------+---------------+-------------------+----------------+ +.. list-table:: + :header-rows: 1 + + * - Feature + - Deprecated in + - Projected removal + - PEP/issue + * - ``typing.io`` and ``typing.re`` submodules + - 3.8 + - 3.13 + - :issue:`38291` + * - ``typing`` versions of standard collections + - 3.9 + - Undecided (see :ref:`deprecated-aliases` for more information) + - :pep:`585` + * - :class:`typing.ByteString` + - 3.9 + - 3.14 + - :gh:`91896` + * - :data:`typing.Text` + - 3.11 + - Undecided + - :gh:`92332` + * - :class:`typing.Hashable` and :class:`typing.Sized` + - 3.12 + - Undecided + - :gh:`94309` + * - :data:`typing.TypeAlias` + - 3.12 + - Undecided + - :pep:`695` diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 5a9a53f83dace0..e1aa4ebb0964dd 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -23,9 +23,9 @@ to an absolute URL given a "base URL." The module has been designed to match the internet RFC on Relative Uniform Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, ``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``mailto``, ``mms``, -``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtspu``, ``sftp``, -``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, ``telnet``, -``wais``, ``ws``, ``wss``. +``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtsps``, ``rtspu``, +``sftp``, ``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, +``telnet``, ``wais``, ``ws``, ``wss``. The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 981020b13cd988..8cee85b32d2a83 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -303,115 +303,18 @@ the Python interpreter registers the ``.pyz`` and ``.pyzw`` file extensions when installed. -Making a Windows executable -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -On Windows, registration of the ``.pyz`` extension is optional, and -furthermore, there are certain places that don't recognise registered -extensions "transparently" (the simplest example is that -``subprocess.run(['myapp'])`` won't find your application - you need to -explicitly specify the extension). - -On Windows, therefore, it is often preferable to create an executable from the -zipapp. This is relatively easy, although it does require a C compiler. The -basic approach relies on the fact that zipfiles can have arbitrary data -prepended, and Windows exe files can have arbitrary data appended. So by -creating a suitable launcher and tacking the ``.pyz`` file onto the end of it, -you end up with a single-file executable that runs your application. - -A suitable launcher can be as simple as the following:: - - #define Py_LIMITED_API 1 - #include "Python.h" - - #define WIN32_LEAN_AND_MEAN - #include - - #ifdef WINDOWS - int WINAPI wWinMain( - HINSTANCE hInstance, /* handle to current instance */ - HINSTANCE hPrevInstance, /* handle to previous instance */ - LPWSTR lpCmdLine, /* pointer to command line */ - int nCmdShow /* show state of window */ - ) - #else - int wmain() - #endif - { - wchar_t **myargv = _alloca((__argc + 1) * sizeof(wchar_t*)); - myargv[0] = __wargv[0]; - memcpy(myargv + 1, __wargv, __argc * sizeof(wchar_t *)); - return Py_Main(__argc+1, myargv); - } - -If you define the ``WINDOWS`` preprocessor symbol, this will generate a -GUI executable, and without it, a console executable. - -To compile the executable, you can either just use the standard MSVC -command line tools, or you can take advantage of the fact that distutils -knows how to compile Python source:: - - >>> from distutils.ccompiler import new_compiler - >>> import distutils.sysconfig - >>> import sys - >>> import os - >>> from pathlib import Path - - >>> def compile(src): - >>> src = Path(src) - >>> cc = new_compiler() - >>> exe = src.stem - >>> cc.add_include_dir(distutils.sysconfig.get_python_inc()) - >>> cc.add_library_dir(os.path.join(sys.base_exec_prefix, 'libs')) - >>> # First the CLI executable - >>> objs = cc.compile([str(src)]) - >>> cc.link_executable(objs, exe) - >>> # Now the GUI executable - >>> cc.define_macro('WINDOWS') - >>> objs = cc.compile([str(src)]) - >>> cc.link_executable(objs, exe + 'w') - - >>> if __name__ == "__main__": - >>> compile("zastub.c") - -The resulting launcher uses the "Limited ABI", so it will run unchanged with -any version of Python 3.x. All it needs is for Python (``python3.dll``) to be -on the user's ``PATH``. - -For a fully standalone distribution, you can distribute the launcher with your -application appended, bundled with the Python "embedded" distribution. This -will run on any PC with the appropriate architecture (32 bit or 64 bit). - - Caveats ~~~~~~~ -There are some limitations to the process of bundling your application into -a single file. In most, if not all, cases they can be addressed without -needing major changes to your application. - -1. If your application depends on a package that includes a C extension, that - package cannot be run from a zip file (this is an OS limitation, as executable - code must be present in the filesystem for the OS loader to load it). In this - case, you can exclude that dependency from the zipfile, and either require - your users to have it installed, or ship it alongside your zipfile and add code - to your ``__main__.py`` to include the directory containing the unzipped - module in ``sys.path``. In this case, you will need to make sure to ship - appropriate binaries for your target architecture(s) (and potentially pick the - correct version to add to ``sys.path`` at runtime, based on the user's machine). - -2. If you are shipping a Windows executable as described above, you either need to - ensure that your users have ``python3.dll`` on their PATH (which is not the - default behaviour of the installer) or you should bundle your application with - the embedded distribution. - -3. The suggested launcher above uses the Python embedding API. This means that in - your application, ``sys.executable`` will be your application, and *not* a - conventional Python interpreter. Your code and its dependencies need to be - prepared for this possibility. For example, if your application uses the - :mod:`multiprocessing` module, it will need to call - :func:`multiprocessing.set_executable` to let the module know where to find the - standard Python interpreter. +If your application depends on a package that includes a C extension, that +package cannot be run from a zip file (this is an OS limitation, as executable +code must be present in the filesystem for the OS loader to load it). In this +case, you can exclude that dependency from the zipfile, and either require +your users to have it installed, or ship it alongside your zipfile and add code +to your ``__main__.py`` to include the directory containing the unzipped +module in ``sys.path``. In this case, you will need to make sure to ship +appropriate binaries for your target architecture(s) (and potentially pick the +correct version to add to ``sys.path`` at runtime, based on the user's machine). The Python Zip Application Archive Format diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b97a08f25d92a2..08dcc8095bee8c 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -298,27 +298,27 @@ Dictionary displays .. index:: pair: dictionary; display pair: dictionary; comprehensions - key, datum, key/datum pair + key, value, key/value pair pair: object; dictionary single: {} (curly brackets); dictionary expression single: : (colon); in dictionary expressions single: , (comma); in dictionary displays -A dictionary display is a possibly empty series of key/datum pairs enclosed in -curly braces: +A dictionary display is a possibly empty series of dict items (key/value pairs) +enclosed in curly braces: .. productionlist:: python-grammar - dict_display: "{" [`key_datum_list` | `dict_comprehension`] "}" - key_datum_list: `key_datum` ("," `key_datum`)* [","] - key_datum: `expression` ":" `expression` | "**" `or_expr` + dict_display: "{" [`dict_item_list` | `dict_comprehension`] "}" + dict_item_list: `dict_item` ("," `dict_item`)* [","] + dict_item: `expression` ":" `expression` | "**" `or_expr` dict_comprehension: `expression` ":" `expression` `comp_for` A dictionary display yields a new dictionary object. -If a comma-separated sequence of key/datum pairs is given, they are evaluated +If a comma-separated sequence of dict items is given, they are evaluated from left to right to define the entries of the dictionary: each key object is -used as a key into the dictionary to store the corresponding datum. This means -that you can specify the same key multiple times in the key/datum list, and the +used as a key into the dictionary to store the corresponding value. This means +that you can specify the same key multiple times in the dict item list, and the final dictionary's value for that key will be the last one given. .. index:: @@ -328,7 +328,7 @@ final dictionary's value for that key will be the last one given. A double asterisk ``**`` denotes :dfn:`dictionary unpacking`. Its operand must be a :term:`mapping`. Each mapping item is added to the new dictionary. Later values replace values already set by -earlier key/datum pairs and earlier dictionary unpackings. +earlier dict items and earlier dictionary unpackings. .. versionadded:: 3.5 Unpacking into dictionary displays, originally proposed by :pep:`448`. @@ -344,7 +344,7 @@ in the new dictionary in the order they are produced. Restrictions on the types of the key values are listed earlier in section :ref:`types`. (To summarize, the key type should be :term:`hashable`, which excludes all mutable objects.) Clashes between duplicate keys are not detected; the last -datum (textually rightmost in the display) stored for a given key value +value (textually rightmost in the display) stored for a given key value prevails. .. versionchanged:: 3.8 diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 662a4b643c4378..a9e65be1eda340 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -210,7 +210,7 @@ Assignment of an object to a single target is recursively defined as follows. If the primary is a mapping object (such as a dictionary), the subscript must have a type compatible with the mapping's key type, and the mapping is then - asked to create a key/datum pair which maps the subscript to the assigned + asked to create a key/value pair which maps the subscript to the assigned object. This can either replace an existing key/value pair with the same key value, or insert a new key/value pair (if no key with the same value existed). diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 9cbd15c2209dc6..4c9d02ea37ab58 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -10,7 +10,6 @@ sphinx==4.5.0 blurb -sphinx-lint==0.6.7 sphinxext-opengraph==0.7.5 # The theme used by the documentation is stored separately, so we need diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 3a5b26f7779618..003229d8a59754 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -14,29 +14,27 @@ from os import getenv, path from time import asctime from pprint import pformat + +from docutils import nodes, utils from docutils.io import StringOutput from docutils.parsers.rst import Directive from docutils.utils import new_document - -from docutils import nodes, utils - from sphinx import addnodes from sphinx.builders import Builder -try: - from sphinx.errors import NoUri -except ImportError: - from sphinx.environment import NoUri +from sphinx.domains.python import PyFunction, PyMethod +from sphinx.errors import NoUri from sphinx.locale import _ as sphinx_gettext -from sphinx.util import status_iterator, logging +from sphinx.util import logging from sphinx.util.docutils import SphinxDirective from sphinx.util.nodes import split_explicit_title from sphinx.writers.text import TextWriter, TextTranslator try: - from sphinx.domains.python import PyFunction, PyMethod + # Sphinx 6+ + from sphinx.util.display import status_iterator except ImportError: - from sphinx.domains.python import PyClassmember as PyMethod - from sphinx.domains.python import PyModulelevel as PyFunction + # Deprecated in Sphinx 6.1, will be removed in Sphinx 8 + from sphinx.util import status_iterator ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' @@ -561,6 +559,7 @@ def finish(self): try: f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8')) f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8')) + f.write('# as part of the release process.\n'.encode('utf-8')) f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8')) finally: f.close() diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index b91f8138553e62..10cb6fd880fa1a 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,6 +26,7 @@ {% endblock %} {% block extrahead %} + {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} diff --git a/Doc/tools/templates/search.html b/Doc/tools/templates/search.html index f2ac2ea0f09873..852974461380f2 100644 --- a/Doc/tools/templates/search.html +++ b/Doc/tools/templates/search.html @@ -1,48 +1,62 @@ {% extends "!search.html" %} {% block extrahead %} {{ super() }} + -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index c9b3d982c31c9a..e140f51f1dda78 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -307,8 +307,9 @@ you can use the class name followed by an argument list resembling a constructor, but with the ability to capture attributes into variables:: class Point: - x: int - y: int + def __init__(self, x, y): + self.x = x + self.y = y def where_is(point): match point: @@ -342,7 +343,13 @@ Dotted names (like ``foo.bar``), attribute names (the ``x=`` and ``y=`` above) o (recognized by the "(...)" next to them like ``Point`` above) are never assigned to. Patterns can be arbitrarily nested. For example, if we have a short -list of points, we could match it like this:: +list of Points, with ``__match_args__`` added, we could match it like this:: + + class Point: + __match_args__ = ('x', 'y') + def __init__(self, x, y): + self.x = x + self.y = y match points: case []: diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index ca5dc3314c63b6..6419ff621f1b31 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -578,6 +578,8 @@ the following pattern:: ... +.. _tut-exception-notes: + Enriching Exceptions with Notes =============================== diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 9d4042ce5a7e8a..1b470d395d6d58 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -109,7 +109,7 @@ source. Many standard library modules contain code that is invoked on their execution as a script. An example is the :mod:`timeit` module:: - python -m timeit -s 'setup here' 'benchmarked code here' + python -m timeit -s "setup here" "benchmarked code here" python -m timeit -h # for details .. audit-event:: cpython.run_module module-name cmdoption-m diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8bc7e8a630e2a6..d6d7b7dac9a310 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -253,6 +253,12 @@ Inlining does result in a few visible behavior changes: * Calling :func:`locals` inside a comprehension now includes variables from outside the comprehension, and no longer includes the synthetic ``.0`` variable for the comprehension "argument". +* A comprehension iterating directly over ``locals()`` (e.g. ``[k for k in + locals()]``) may see "RuntimeError: dictionary changed size during iteration" + when run under tracing (e.g. code coverage measurement). This is the same + behavior already seen in e.g. ``for k in locals():``. To avoid the error, first + create a list of keys to iterate over: ``keys = list(locals()); [k for k in + keys]``. Contributed by Carl Meyer and Vladimir Matveev in :pep:`709`. @@ -564,6 +570,14 @@ dis :data:`~dis.hasarg` collection instead. (Contributed by Irit Katriel in :gh:`94216`.) +email +----- + +* :func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now return + ``('', '')`` 2-tuples in more situations where invalid email addresses are + encountered instead of potentially inaccurate values. + (Contributed by Thomas Dwyer for :gh:`102988` to ameliorate CVE-2023-27043.) + fractions --------- @@ -593,14 +607,6 @@ itertools tuples where the last batch may be shorter than the rest. (Contributed by Raymond Hettinger in :gh:`98363`.) -json ----- - -* Added :class:`json.AttrDict` for use with ``object_hook`` in :func:`json.load` - or :func:`json.loads`. This is a subclass of :class:`dict` that also supports - attribute style dotted access. - (Contributed by Raymond Hettinger in :gh:`96145`.) - math ---- @@ -1368,6 +1374,53 @@ Removed * The :mod:`!imp` module has been removed. (Contributed by Barry Warsaw in :gh:`98040`.) + * Replace removed :mod:`!imp` functions with :mod:`importlib` functions: + + ================================= ======================================= + imp importlib + ================================= ======================================= + ``imp.NullImporter`` Insert ``None`` into ``sys.path_importer_cache`` + ``imp.cache_from_source()`` :func:`importlib.util.cache_from_source` + ``imp.find_module()`` :func:`importlib.util.find_spec` + ``imp.get_magic()`` :attr:`importlib.util.MAGIC_NUMBER` + ``imp.get_suffixes()`` :attr:`importlib.machinery.SOURCE_SUFFIXES`, :attr:`importlib.machinery.EXTENSION_SUFFIXES`, and :attr:`importlib.machinery.BYTECODE_SUFFIXES` + ``imp.get_tag()`` :attr:`sys.implementation.cache_tag ` + ``imp.load_module()`` :func:`importlib.import_module` + ``imp.new_module(name)`` ``types.ModuleType(name)`` + ``imp.reload()`` :func:`importlib.reload` + ``imp.source_from_cache()`` :func:`importlib.util.source_from_cache` + ================================= ======================================= + + * Replace ``imp.load_source()`` with:: + + import importlib.util + import importlib.machinery + + def load_source(modname, filename): + loader = importlib.machinery.SourceFileLoader(modname, filename) + spec = importlib.util.spec_from_file_location(modname, filename, loader=loader) + module = importlib.util.module_from_spec(spec) + # The module is always executed and not cached in sys.modules. + # Uncomment the following line to cache the module. + # sys.modules[module.__name__] = module + loader.exec_module(module) + return module + + * Removed :mod:`!imp` functions and attributes with no replacements: + + * undocumented functions: + + * ``imp.init_builtin()`` + * ``imp.load_compiled()`` + * ``imp.load_dynamic()`` + * ``imp.load_package()`` + + * ``imp.lock_held()``, ``imp.acquire_lock()``, ``imp.release_lock()``: + the locking scheme has changed in Python 3.3 to per-module locks. + * ``imp.find_module()`` constants: ``SEARCH_ERROR``, ``PY_SOURCE``, + ``PY_COMPILED``, ``C_EXTENSION``, ``PY_RESOURCE``, ``PKG_DIRECTORY``, + ``C_BUILTIN``, ``PY_FROZEN``, ``PY_CODERESOURCE``, ``IMP_HOOK``. + * Removed the ``suspicious`` rule from the documentation Makefile, and removed ``Doc/tools/rstlint.py``, both in favor of `sphinx-lint `_. @@ -1498,6 +1551,9 @@ Changes in the Python API * Some incomplete or invalid Python code now raises :exc:`tokenize.TokenError` instead of returning arbitrary ``ERRORTOKEN`` tokens when tokenizing it. + * Mixing tabs and spaces as indentation in the same file is not supported anymore and will + raise a :exc:`TabError`. + Build Changes ============= @@ -1688,6 +1744,11 @@ New Features (Contributed by Eddie Elizondo in :gh:`84436`.) +* In the limited C API version 3.12, :c:func:`Py_INCREF` and + :c:func:`Py_DECREF` functions are now implemented as opaque function calls to + hide implementation details. + (Contributed by Victor Stinner in :gh:`105387`.) + Porting to Python 3.12 ---------------------- @@ -1775,7 +1836,31 @@ Porting to Python 3.12 allowing incomplete initialization. Note that :c:func:`PyType_FromMetaclass` (added in Python 3.12) - already disallows creating classes whose metaclass overrides ``tp_new``. + already disallows creating classes whose metaclass overrides ``tp_new`` + (:meth:`~object.__new__` in Python). + + Since ``tp_new`` overrides almost everything ``PyType_From*`` functions do, + the two are incompatible with each other. + The existing behavior -- ignoring the metaclass for several steps + of type creation -- is unsafe in general, since (meta)classes assume that + ``tp_new`` was called. + There is no simple general workaround. One of the following may work for you: + + - If you control the metaclass, avoid using ``tp_new`` in it: + + - If initialization can be skipped, it can be done in + :c:member:`~PyTypeObject.tp_init` instead. + - If the metaclass doesn't need to be instantiated from Python, + set its ``tp_new`` to ``NULL`` using + the :const:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag. + This makes it acceptable for ``PyType_From*`` functions. + + - Avoid ``PyType_From*`` functions: if you don't need C-specific features + (slots or setting the instance size), create types by :ref:`calling ` + the metaclass. + + - If you *know* the ``tp_new`` can be skipped safely, filter the deprecation + warning out using :func:`warnings.catch_warnings` from Python. * :c:var:`PyOS_InputHook` and :c:var:`PyOS_ReadlineFunctionPointer` are no longer called in :ref:`subinterpreters `. This is @@ -1810,6 +1895,7 @@ Deprecated * :c:var:`Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` * :c:var:`Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` * :c:var:`!Py_FileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` + * :c:var:`!Py_HasFileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` * :c:var:`!Py_FileSystemDefaultEncodeErrors`: use :c:member:`PyConfig.filesystem_errors` * :c:var:`!Py_UTF8Mode`: use :c:member:`PyPreConfig.utf8_mode` (see :c:func:`Py_PreInitialize`) diff --git a/Grammar/python.gram b/Grammar/python.gram index 6b2a46aff0dcf0..c1863aec67cc2b 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -1293,7 +1293,7 @@ invalid_group: | '(' a='**' expression ')' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use double starred expression here") } invalid_import: - | a='import' dotted_name 'from' dotted_name { + | a='import' ','.dotted_name+ 'from' dotted_name { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "Did you mean to use 'from ... import ...' instead?") } invalid_import_from_targets: diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 0fbbee10c2edce..a9616bd6a4f518 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -22,6 +22,8 @@ PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _P PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds); PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void); +PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *); + PyAPI_FUNC(Py_ssize_t) PyUnstable_Eval_RequestCodeExtraIndex(freefunc); // Old name -- remove when this API changes: _Py_DEPRECATED_EXTERNALLY(3.12) static inline Py_ssize_t diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index efae2409b50069..c103c2026e40e9 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -244,6 +244,10 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config, /* --- PyInterpreterConfig ------------------------------------ */ +#define PyInterpreterConfig_DEFAULT_GIL (0) +#define PyInterpreterConfig_SHARED_GIL (1) +#define PyInterpreterConfig_OWN_GIL (2) + typedef struct { // XXX "allow_object_sharing"? "own_objects"? int use_main_obmalloc; @@ -252,7 +256,7 @@ typedef struct { int allow_threads; int allow_daemon_threads; int check_multi_interp_extensions; - int own_gil; + int gil; } PyInterpreterConfig; #define _PyInterpreterConfig_INIT \ @@ -263,7 +267,7 @@ typedef struct { .allow_threads = 1, \ .allow_daemon_threads = 0, \ .check_multi_interp_extensions = 1, \ - .own_gil = 1, \ + .gil = PyInterpreterConfig_OWN_GIL, \ } #define _PyInterpreterConfig_LEGACY_INIT \ @@ -274,7 +278,7 @@ typedef struct { .allow_threads = 1, \ .allow_daemon_threads = 1, \ .check_multi_interp_extensions = 0, \ - .own_gil = 0, \ + .gil = PyInterpreterConfig_SHARED_GIL, \ } /* --- Helper functions --------------------------------------- */ diff --git a/Include/cpython/object.h b/Include/cpython/object.h index d8eff691039d24..c5d0851a4b11bf 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -283,6 +283,7 @@ PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject * PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *); PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *); PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); +PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); diff --git a/Include/fileobject.h b/Include/fileobject.h index 02bd7c915a23f7..2deef544d667a5 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -23,7 +23,7 @@ Py_DEPRECATED(3.12) PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000 Py_DEPRECATED(3.12) PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors; #endif -PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; +Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_UTF8Mode; diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index ca2703781de4b0..fc0f72efdae48e 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -27,7 +27,8 @@ PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp); PyAPI_FUNC(int) _PyEval_AddPendingCall( PyInterpreterState *interp, int (*func)(void *), - void *arg); + void *arg, + int mainthreadonly); PyAPI_FUNC(void) _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); @@ -153,6 +154,7 @@ extern PyObject* _Py_MakeCoro(PyFunctionObject *func); extern int _Py_HandlePending(PyThreadState *tstate); +extern PyObject * _PyEval_GetFrameLocals(void); #ifdef __cplusplus diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 95d1fa16ba40dc..e56e43c6e0c6a7 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -13,6 +13,24 @@ extern "C" { #include "pycore_gil.h" // struct _gil_runtime_state +struct _pending_calls { + int busy; + PyThread_type_lock lock; + /* Request for running pending calls. */ + _Py_atomic_int calls_to_do; + /* Request for looking at the `async_exc` field of the current + thread state. + Guarded by the GIL. */ + int async_exc; +#define NPENDINGCALLS 32 + struct _pending_call { + int (*func)(void *); + void *arg; + } calls[NPENDINGCALLS]; + int first; + int last; +}; + typedef enum { PERF_STATUS_FAILED = -1, // Perf trampoline is in an invalid state PERF_STATUS_NO_INIT = 0, // Perf trampoline is not initialized @@ -49,6 +67,8 @@ struct _ceval_runtime_state { the main thread of the main interpreter can handle signals: see _Py_ThreadCanHandleSignals(). */ _Py_atomic_int signals_pending; + /* Pending calls to be made only on the main thread. */ + struct _pending_calls pending_mainthread; }; #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -62,24 +82,6 @@ struct _ceval_runtime_state { #endif -struct _pending_calls { - int busy; - PyThread_type_lock lock; - /* Request for running pending calls. */ - _Py_atomic_int calls_to_do; - /* Request for looking at the `async_exc` field of the current - thread state. - Guarded by the GIL. */ - int async_exc; -#define NPENDINGCALLS 32 - struct { - int (*func)(void *); - void *arg; - } calls[NPENDINGCALLS]; - int first; - int last; -}; - struct _ceval_state { /* This single variable consolidates all requests to break out of the fast path in the eval loop. */ diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index a72e03f1438fc8..158db2cf9df82e 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -226,6 +226,9 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden); + int _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame); diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index d5819fcd1c5038..439f47a263dfa1 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -12,9 +12,8 @@ extern "C" { static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { if (Py_REFCNT(obj) < _Py_IMMORTAL_REFCNT) { - _PyObject_ASSERT_FAILED_MSG(obj, - "immortal object has less refcnt than expected " - "_Py_IMMORTAL_REFCNT"); + fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); + _PyObject_Dump(obj); } } #endif diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 04474c1da8855a..619225c939664e 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -48,12 +48,22 @@ struct _Py_long_state { */ struct _is { - struct _ceval_state ceval; PyInterpreterState *next; + int64_t id; + int64_t id_refcount; + int requires_idref; + PyThread_type_lock id_mutex; + + /* Has been initialized to a safe state. + + In order to be effective, this must be set to 0 during or right + after allocation. */ + int _initialized; + int finalizing; + uint64_t monitoring_version; uint64_t last_restart_version; - struct pythreads { uint64_t next_unique_id; /* The linked list of threads, newest first. */ @@ -72,18 +82,6 @@ struct _is { Get runtime from tstate: tstate->interp->runtime. */ struct pyruntimestate *runtime; - int64_t id; - int64_t id_refcount; - int requires_idref; - PyThread_type_lock id_mutex; - - /* Has been initialized to a safe state. - - In order to be effective, this must be set to 0 during or right - after allocation. */ - int _initialized; - int finalizing; - /* Set by Py_EndInterpreter(). Use _PyInterpreterState_GetFinalizing() @@ -91,17 +89,38 @@ struct _is { to access it, don't access it directly. */ _Py_atomic_address _finalizing; - struct _obmalloc_state obmalloc; - struct _gc_runtime_state gc; - struct _import_state imports; + /* The following fields are here to avoid allocation during init. + The data is exposed through PyInterpreterState pointer fields. + These fields should not be accessed directly outside of init. + + All other PyInterpreterState pointer fields are populated when + needed and default to NULL. + + For now there are some exceptions to that rule, which require + allocation during init. These will be addressed on a case-by-case + basis. Also see _PyRuntimeState regarding the various mutex fields. + */ // Dictionary of the sys module PyObject *sysdict; + // Dictionary of the builtins module PyObject *builtins; + struct _ceval_state ceval; + + struct _import_state imports; + + /* The per-interpreter GIL, which might not be used. */ + struct _gil_runtime_state _gil; + + /* ---------- IMPORTANT --------------------------- + The fields above this line are declared as early as + possible to facilitate out-of-process observability + tools. */ + PyObject *codec_search_path; PyObject *codec_search_cache; PyObject *codec_error_registry; @@ -133,6 +152,8 @@ struct _is { struct _warnings_runtime_state warnings; struct atexit_state atexit; + struct _obmalloc_state obmalloc; + PyObject *audit_hooks; PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; PyCode_WatchCallback code_watchers[CODE_MAX_WATCHERS]; @@ -173,22 +194,7 @@ struct _is { struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; - /* The following fields are here to avoid allocation during init. - The data is exposed through PyInterpreterState pointer fields. - These fields should not be accessed directly outside of init. - - All other PyInterpreterState pointer fields are populated when - needed and default to NULL. - - For now there are some exceptions to that rule, which require - allocation during init. These will be addressed on a case-by-case - basis. Also see _PyRuntimeState regarding the various mutex fields. - */ - - /* The per-interpreter GIL, which might not be used. */ - struct _gil_runtime_state _gil; - - /* the initial PyInterpreterState.threads.head */ + /* the initial PyInterpreterState.threads.head */ PyThreadState _initial_thread; }; diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index daa40cf4bcd855..43652c4405ec1a 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -60,14 +60,6 @@ _Py_ThreadCanHandleSignals(PyInterpreterState *interp) } -/* Only execute pending calls on the main thread. */ -static inline int -_Py_ThreadCanHandlePendingCalls(void) -{ - return _Py_IsMainThread(); -} - - /* Variable and static inline functions for in-line access to current thread and interpreter state */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 8f51e2def69fc9..5ed97e9715b2b0 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -84,13 +84,6 @@ typedef struct pyruntimestate { to access it, don't access it directly. */ _Py_atomic_address _finalizing; - struct _pymem_allocators allocators; - struct _obmalloc_global_state obmalloc; - struct pyhash_runtime_state pyhash_state; - struct _time_runtime_state time; - struct _pythread_runtime_state threads; - struct _signals_runtime_state signals; - struct pyinterpreters { PyThread_type_lock mutex; /* The linked list of interpreters, newest first. */ @@ -109,13 +102,26 @@ typedef struct pyruntimestate { using a Python int. */ int64_t next_id; } interpreters; + + unsigned long main_thread; + + /* ---------- IMPORTANT --------------------------- + The fields above this line are declared as early as + possible to facilitate out-of-process observability + tools. */ + // XXX Remove this field once we have a tp_* slot. struct _xidregistry { PyThread_type_lock mutex; struct _xidregitem *head; } xidregistry; - unsigned long main_thread; + struct _pymem_allocators allocators; + struct _obmalloc_global_state obmalloc; + struct pyhash_runtime_state pyhash_state; + struct _time_runtime_state time; + struct _pythread_runtime_state threads; + struct _signals_runtime_state signals; /* Used for the thread state bound to the current thread. */ Py_tss_t autoTSSkey; diff --git a/Include/object.h b/Include/object.h index ad16b72cd42474..3ef64511399c66 100644 --- a/Include/object.h +++ b/Include/object.h @@ -610,10 +610,11 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *); static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { -#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) - // Stable ABI for Python built in debug mode. _Py_IncRef() was added to - // Python 3.10.0a7, use Py_IncRef() on older Python versions. Py_IncRef() - // accepts NULL whereas _Py_IncRef() doesn't. +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) + // Stable ABI implements Py_INCREF() as a function call on limited C API + // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() + // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. + // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. # if Py_LIMITED_API+0 >= 0x030a00A7 _Py_IncRef(op); # else @@ -647,10 +648,11 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) # define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) #endif -#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) -// Stable ABI for Python built in debug mode. _Py_DecRef() was added to Python -// 3.10.0a7, use Py_DecRef() on older Python versions. Py_DecRef() accepts NULL -// whereas _Py_IncRef() doesn't. +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +// Stable ABI implements Py_DECREF() as a function call on limited C API +// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was +// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. +// Py_DecRef() accepts NULL whereas _Py_IncRef() doesn't. static inline void Py_DECREF(PyObject *op) { # if Py_LIMITED_API+0 >= 0x030a00A7 _Py_DecRef(op); diff --git a/Include/patchlevel.h b/Include/patchlevel.h index a6d20e4d03cf58..e3d7ef51ba60c0 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 12 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 2 +#define PY_RELEASE_SERIAL 4 /* Version as a string */ -#define PY_VERSION "3.12.0b2+" +#define PY_VERSION "3.12.0b4+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/ast.py b/Lib/ast.py index 226910ecac0b4a..a307f3ecd06175 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1051,7 +1051,8 @@ def visit_ClassDef(self, node): self.fill("@") self.traverse(deco) self.fill("class " + node.name) - self._type_params_helper(node.type_params) + if hasattr(node, "type_params"): + self._type_params_helper(node.type_params) with self.delimit_if("(", ")", condition = node.bases or node.keywords): comma = False for e in node.bases: @@ -1083,7 +1084,8 @@ def _function_helper(self, node, fill_suffix): self.traverse(deco) def_str = fill_suffix + " " + node.name self.fill(def_str) - self._type_params_helper(node.type_params) + if hasattr(node, "type_params"): + self._type_params_helper(node.type_params) with self.delimit("(", ")"): self.traverse(node.args) if node.returns: diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index fed16ec7c67fac..03165a425eb7d2 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -34,6 +34,7 @@ streams.__all__ + subprocess.__all__ + tasks.__all__ + + taskgroups.__all__ + threads.__all__ + timeouts.__all__ + transports.__all__) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 32d7e1c481ecc5..259004650d5ab3 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -727,7 +727,7 @@ def call_later(self, delay, callback, *args, context=None): always relative to the current time. Each callback will be called exactly once. If two callbacks - are scheduled for exactly the same time, it undefined which + are scheduled for exactly the same time, it is undefined which will be called first. Any positional arguments after the callback will be passed to diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index ce44942186b272..0ccf85105e6673 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -617,7 +617,7 @@ class AbstractEventLoopPolicy: def get_event_loop(self): """Get the event loop for the current context. - Returns an event loop object implementing the BaseEventLoop interface, + Returns an event loop object implementing the AbstractEventLoop interface, or raises an exception in case no event loop has been set for the current context and the current policy does not specify to create one. diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index fa2422b7fba4a7..f895750e3cf959 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1202,6 +1202,7 @@ def _reset_empty_waiter(self): def close(self): self._read_ready_cb = None + self._write_ready = None super().close() diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 06b2e0db86a1fe..930da53d908e36 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -2,7 +2,7 @@ # license: PSFL. -__all__ = ["TaskGroup"] +__all__ = ("TaskGroup",) from . import events from . import exceptions diff --git a/Lib/colorsys.py b/Lib/colorsys.py index 9bdc83e3772603..bc897bd0f99298 100644 --- a/Lib/colorsys.py +++ b/Lib/colorsys.py @@ -83,7 +83,7 @@ def rgb_to_hls(r, g, b): if l <= 0.5: s = rangec / sumc else: - s = rangec / (2.0-sumc) + s = rangec / (2.0-maxc-minc) # Not always 2.0-sumc: gh-106498. rc = (maxc-r) / rangec gc = (maxc-g) / rangec bc = (maxc-b) / rangec diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 816edab99f63e3..301207f59de37a 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -499,6 +499,10 @@ def terminate_broken(self, cause): for p in self.processes.values(): p.terminate() + # Prevent queue writing to a pipe which is no longer read. + # https://github.com/python/cpython/issues/94777 + self.call_queue._reader.close() + # clean up resources self.join_executor_internals() diff --git a/Lib/configparser.py b/Lib/configparser.py index dee5a0db7e7ddc..e8aae217949a9f 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -59,7 +59,7 @@ instance. It will be used as the handler for option value pre-processing when using getters. RawConfigParser objects don't do any sort of interpolation, whereas ConfigParser uses an instance of - BasicInterpolation. The library also provides a ``zc.buildbot`` + BasicInterpolation. The library also provides a ``zc.buildout`` inspired ExtendedInterpolation implementation. When `converters` is given, it should be a dictionary where each key diff --git a/Lib/datetime.py b/Lib/datetime.py index bad8beb4f6b026..a33d2d724cb33d 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -6,4 +6,4 @@ from _pydatetime import __doc__ __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", - "MINYEAR", "MAXYEAR") + "MINYEAR", "MAXYEAR", "UTC") diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 81da5394ea1695..11ad75e94e9345 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -106,12 +106,54 @@ def formataddr(pair, charset='utf-8'): return address +def _pre_parse_validation(email_header_fields): + accepted_values = [] + for v in email_header_fields: + s = v.replace('\\(', '').replace('\\)', '') + if s.count('(') != s.count(')'): + v = "('', '')" + accepted_values.append(v) + + return accepted_values + + +def _post_parse_validation(parsed_email_header_tuples): + accepted_values = [] + # The parser would have parsed a correctly formatted domain-literal + # The existence of an [ after parsing indicates a parsing failure + for v in parsed_email_header_tuples: + if '[' in v[1]: + v = ('', '') + accepted_values.append(v) + + return accepted_values + def getaddresses(fieldvalues): - """Return a list of (REALNAME, EMAIL) for each fieldvalue.""" - all = COMMASPACE.join(str(v) for v in fieldvalues) + """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue. + + When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in + its place. + + If the resulting list of parsed address is not the same as the number of + fieldvalues in the input list a parsing error has occurred. A list + containing a single empty 2-tuple [('', '')] is returned in its place. + This is done to avoid invalid output. + """ + fieldvalues = [str(v) for v in fieldvalues] + fieldvalues = _pre_parse_validation(fieldvalues) + all = COMMASPACE.join(v for v in fieldvalues) a = _AddressList(all) - return a.addresslist + result = _post_parse_validation(a.addresslist) + + n = 0 + for v in fieldvalues: + n += v.count(',') + 1 + + if len(result) != n: + return [('', '')] + + return result def _format_timetuple_and_zone(timetuple, zone): @@ -212,9 +254,18 @@ def parseaddr(addr): Return a tuple of realname and email address, unless the parse fails, in which case return a 2-tuple of ('', ''). """ - addrs = _AddressList(addr).addresslist - if not addrs: - return '', '' + if isinstance(addr, list): + addr = addr[0] + + if not isinstance(addr, str): + return ('', '') + + addr = _pre_parse_validation([addr])[0] + addrs = _post_parse_validation(_AddressList(addr).addresslist) + + if not addrs or len(addrs) > 1: + return ('', '') + return addrs[0] diff --git a/Lib/enum.py b/Lib/enum.py index 62df304057a108..9114c5ccaaa95c 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1242,6 +1242,12 @@ def __hash__(self): def __reduce_ex__(self, proto): return self.__class__, (self._value_, ) + def __deepcopy__(self,memo): + return self + + def __copy__(self): + return self + # enum.property is used to provide access to the `name` and # `value` attributes of enum members while keeping some measure of # protection from modification, while still allowing for an enumeration @@ -1463,12 +1469,11 @@ def _missing_(cls, value): else: pseudo_member._name_ = None # use setdefault in case another thread already created a composite - # with this value, but only if all members are known - # note: zero is a special case -- add it - if not unknown: - pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) - if neg_value is not None: - cls._value2member_map_[neg_value] = pseudo_member + # with this value + # note: zero is a special case -- always add it + pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) + if neg_value is not None: + cls._value2member_map_[neg_value] = pseudo_member return pseudo_member def __contains__(self, other): @@ -1540,14 +1545,10 @@ def __xor__(self, other): def __invert__(self): if self._inverted_ is None: - if self._boundary_ is KEEP: - # use all bits + if self._boundary_ in (EJECT, KEEP): self._inverted_ = self.__class__(~self._value_) else: - # calculate flags not in this member - self._inverted_ = self.__class__(self._flag_mask_ ^ self._value_) - if isinstance(self._inverted_, self.__class__): - self._inverted_._inverted_ = self + self._inverted_ = self.__class__(self._singles_mask_ & ~self._value_) return self._inverted_ __rand__ = __and__ diff --git a/Lib/functools.py b/Lib/functools.py index 72b2103e7a5544..2ae4290f983982 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -956,9 +956,10 @@ def __isabstractmethod__(self): ################################################################################ -### cached_property() - computed once per instance, cached as attribute +### cached_property() - property result cached as instance attribute ################################################################################ +_NOT_FOUND = object() class cached_property: def __init__(self, func): @@ -989,15 +990,17 @@ def __get__(self, instance, owner=None): f"instance to cache {self.attrname!r} property." ) raise TypeError(msg) from None - val = self.func(instance) - try: - cache[self.attrname] = val - except TypeError: - msg = ( - f"The '__dict__' attribute on {type(instance).__name__!r} instance " - f"does not support item assignment for caching {self.attrname!r} property." - ) - raise TypeError(msg) from None + val = cache.get(self.attrname, _NOT_FOUND) + if val is _NOT_FOUND: + val = self.func(instance) + try: + cache[self.attrname] = val + except TypeError: + msg = ( + f"The '__dict__' attribute on {type(instance).__name__!r} instance " + f"does not support item assignment for caching {self.attrname!r} property." + ) + raise TypeError(msg) from None return val __class_getitem__ = classmethod(GenericAlias) diff --git a/Lib/gzip.py b/Lib/gzip.py index 8796c8d9fd9a2d..cf8b675064ce89 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -370,8 +370,9 @@ def close(self): def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): self._check_not_closed() if self.mode == WRITE: - # Ensure the compressor's buffer is flushed self._buffer.flush() + # Ensure the compressor's buffer is flushed + self.fileobj.write(self.compress.flush(zlib_mode)) self.fileobj.flush() def fileno(self): diff --git a/Lib/http/client.py b/Lib/http/client.py index 3d98e4eb54bb45..b35b1d6368aae7 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -970,13 +970,12 @@ def get_proxy_response_headers(self): received from the proxy server to the CONNECT request sent to set the tunnel. - If the CONNECT request was not sent, the method returns - an empty dictionary. + If the CONNECT request was not sent, the method returns None. """ return ( _parse_header_lines(self._raw_proxy_headers) if self._raw_proxy_headers is not None - else {} + else None ) def connect(self): diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 4ffc90ab0c852a..53e80a9b42801f 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -140,11 +140,12 @@ def main(del_exitfunc=False): capture_warnings(True) sys.argv[:] = [""] - sockthread = threading.Thread(target=manage_socket, - name='SockThread', - args=((LOCALHOST, port),)) - sockthread.daemon = True - sockthread.start() + threading.Thread(target=manage_socket, + name='SockThread', + args=((LOCALHOST, port),), + daemon=True, + ).start() + while True: try: if exit_now: diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 256e76a0a67f8f..ed2c74771ea87d 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -97,7 +97,7 @@ """ __version__ = '2.0.9' __all__ = [ - 'dump', 'dumps', 'load', 'loads', 'AttrDict', + 'dump', 'dumps', 'load', 'loads', 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', ] @@ -357,53 +357,3 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, if parse_constant is not None: kw['parse_constant'] = parse_constant return cls(**kw).decode(s) - -class AttrDict(dict): - """Dict like object that supports attribute style dotted access. - - This class is intended for use with the *object_hook* in json.loads(): - - >>> from json import loads, AttrDict - >>> json_string = '{"mercury": 88, "venus": 225, "earth": 365, "mars": 687}' - >>> orbital_period = loads(json_string, object_hook=AttrDict) - >>> orbital_period['earth'] # Dict style lookup - 365 - >>> orbital_period.earth # Attribute style lookup - 365 - >>> orbital_period.keys() # All dict methods are present - dict_keys(['mercury', 'venus', 'earth', 'mars']) - - Attribute style access only works for keys that are valid attribute names. - In contrast, dictionary style access works for all keys. - For example, ``d.two words`` contains a space and is not syntactically - valid Python, so ``d["two words"]`` should be used instead. - - If a key has the same name as dictionary method, then a dictionary - lookup finds the key and an attribute lookup finds the method: - - >>> d = AttrDict(items=50) - >>> d['items'] # Lookup the key - 50 - >>> d.items() # Call the method - dict_items([('items', 50)]) - - """ - __slots__ = () - - def __getattr__(self, attr): - try: - return self[attr] - except KeyError: - raise AttributeError(attr) from None - - def __setattr__(self, attr, value): - self[attr] = value - - def __delattr__(self, attr): - try: - del self[attr] - except KeyError: - raise AttributeError(attr) from None - - def __dir__(self): - return list(self) + dir(type(self)) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 09f8a229d7cccb..f1af7709104714 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -31,11 +31,13 @@ WINSERVICE = False else: WINEXE = getattr(sys, 'frozen', False) - WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") + WINSERVICE = sys.executable and sys.executable.lower().endswith("pythonservice.exe") def set_executable(exe): global _python_exe - if sys.platform == 'win32': + if exe is None: + _python_exe = exe + elif sys.platform == 'win32': _python_exe = os.fsdecode(exe) else: _python_exe = os.fsencode(exe) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index d279fd2958b170..b99bf6e7dd2408 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -127,12 +127,19 @@ def _compile_pattern_lines(pattern_lines, case_sensitive): # Match the start of the path, or just after a path separator parts = ['^'] for part in pattern_lines.splitlines(keepends=True): - # We slice off the common prefix and suffix added by translate() to - # ensure that re.DOTALL is not set, and the end of the string not - # matched, respectively. With DOTALL not set, '*' wildcards will not - # match path separators, because the '.' characters in the pattern - # will not match newlines. - parts.append(fnmatch.translate(part)[_FNMATCH_SLICE]) + if part == '*\n': + part = r'.+\n' + elif part == '*': + part = r'.+' + else: + # Any other component: pass to fnmatch.translate(). We slice off + # the common prefix and suffix added by translate() to ensure that + # re.DOTALL is not set, and the end of the string not matched, + # respectively. With DOTALL not set, '*' wildcards will not match + # path separators, because the '.' characters in the pattern will + # not match newlines. + part = fnmatch.translate(part)[_FNMATCH_SLICE] + parts.append(part) # Match the end of the path, always. parts.append(r'\Z') flags = re.MULTILINE @@ -501,8 +508,12 @@ def _lines(self): try: return self._lines_cached except AttributeError: - trans = _SWAP_SEP_AND_NEWLINE[self._flavour.sep] - self._lines_cached = str(self).translate(trans) + path_str = str(self) + if path_str == '.': + self._lines_cached = '' + else: + trans = _SWAP_SEP_AND_NEWLINE[self._flavour.sep] + self._lines_cached = path_str.translate(trans) return self._lines_cached def __eq__(self, other): diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 84e673a7f87f90..185f09e603df2e 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1780,12 +1780,21 @@ def render_doc(thing, title='Python Library Documentation: %s', forceload=0, return title % desc + '\n\n' + renderer.document(object, name) def doc(thing, title='Python Library Documentation: %s', forceload=0, - output=None): + output=None, is_cli=False): """Display text documentation, given an object or a path to an object.""" if output is None: - pager(render_doc(thing, title, forceload)) + try: + pager(render_doc(thing, title, forceload)) + except ImportError as exc: + if is_cli: + raise + print(exc) else: - output.write(render_doc(thing, title, forceload, plaintext)) + try: + s = render_doc(thing, title, forceload, plaintext) + except ImportError as exc: + s = str(exc) + output.write(s) def writedoc(thing, forceload=0): """Write HTML documentation to a file in the current directory.""" @@ -2044,7 +2053,7 @@ def getline(self, prompt): self.output.flush() return self.input.readline() - def help(self, request): + def help(self, request, is_cli=False): if isinstance(request, str): request = request.strip() if request == 'keywords': self.listkeywords() @@ -2056,13 +2065,13 @@ def help(self, request): elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too - doc(eval(request), 'Help on %s:') + doc(eval(request), 'Help on %s:', is_cli=is_cli) elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) - elif request: doc(request, 'Help on %s:', output=self._output) - else: doc(str, 'Help on %s:', output=self._output) + elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) + else: doc(str, 'Help on %s:', output=self._output, is_cli=is_cli) elif isinstance(request, Helper): self() - else: doc(request, 'Help on %s:', output=self._output) + else: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) self.output.write('\n') def intro(self): @@ -2800,7 +2809,7 @@ class BadUsage(Exception): pass else: writedoc(arg) else: - help.help(arg) + help.help(arg, is_cli=True) except (ImportError, ErrorDuringImport) as value: print(value) sys.exit(1) diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index d70bf9e86b2de2..9603975729cfaf 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Jun 6 16:12:51 2023 +# Autogenerated by Sphinx on Tue Jul 11 14:22:58 2023 +# as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -208,7 +209,7 @@ 'the\n' ' subscript must have a type compatible with the mapping’s key ' 'type,\n' - ' and the mapping is then asked to create a key/datum pair ' + ' and the mapping is then asked to create a key/value pair ' 'which maps\n' ' the subscript to the assigned object. This can either ' 'replace an\n' @@ -5687,30 +5688,31 @@ 'dict': 'Dictionary displays\n' '*******************\n' '\n' - 'A dictionary display is a possibly empty series of key/datum pairs\n' - 'enclosed in curly braces:\n' + 'A dictionary display is a possibly empty series of dict items\n' + '(key/value pairs) enclosed in curly braces:\n' '\n' - ' dict_display ::= "{" [key_datum_list | dict_comprehension] ' + ' dict_display ::= "{" [dict_item_list | dict_comprehension] ' '"}"\n' - ' key_datum_list ::= key_datum ("," key_datum)* [","]\n' - ' key_datum ::= expression ":" expression | "**" or_expr\n' + ' dict_item_list ::= dict_item ("," dict_item)* [","]\n' + ' dict_item ::= expression ":" expression | "**" or_expr\n' ' dict_comprehension ::= expression ":" expression comp_for\n' '\n' 'A dictionary display yields a new dictionary object.\n' '\n' - 'If a comma-separated sequence of key/datum pairs is given, they are\n' + 'If a comma-separated sequence of dict items is given, they are\n' 'evaluated from left to right to define the entries of the ' 'dictionary:\n' 'each key object is used as a key into the dictionary to store the\n' - 'corresponding datum. This means that you can specify the same key\n' - 'multiple times in the key/datum list, and the final dictionary’s ' + 'corresponding value. This means that you can specify the same key\n' + 'multiple times in the dict item list, and the final dictionary’s ' 'value\n' 'for that key will be the last one given.\n' '\n' 'A double asterisk "**" denotes *dictionary unpacking*. Its operand\n' 'must be a *mapping*. Each mapping item is added to the new\n' - 'dictionary. Later values replace values already set by earlier\n' - 'key/datum pairs and earlier dictionary unpackings.\n' + 'dictionary. Later values replace values already set by earlier ' + 'dict\n' + 'items and earlier dictionary unpackings.\n' '\n' 'New in version 3.5: Unpacking into dictionary displays, originally\n' 'proposed by **PEP 448**.\n' @@ -5726,7 +5728,7 @@ 'Restrictions on the types of the key values are listed earlier in\n' 'section The standard type hierarchy. (To summarize, the key type\n' 'should be *hashable*, which excludes all mutable objects.) Clashes\n' - 'between duplicate keys are not detected; the last datum (textually\n' + 'between duplicate keys are not detected; the last value (textually\n' 'rightmost in the display) stored for a given key value prevails.\n' '\n' 'Changed in version 3.8: Prior to Python 3.8, in dict ' @@ -6522,22 +6524,26 @@ 'positional\n' 'argument, and if it’s a keyword, it refers to a named ' 'keyword\n' - 'argument. If the numerical arg_names in a format string ' - 'are 0, 1, 2,\n' - '… in sequence, they can all be omitted (not just some) and ' - 'the numbers\n' - '0, 1, 2, … will be automatically inserted in that order. ' - 'Because\n' - '*arg_name* is not quote-delimited, it is not possible to ' - 'specify\n' - 'arbitrary dictionary keys (e.g., the strings "\'10\'" or ' - '"\':-]\'") within\n' - 'a format string. The *arg_name* can be followed by any ' - 'number of index\n' - 'or attribute expressions. An expression of the form ' - '"\'.name\'" selects\n' - 'the named attribute using "getattr()", while an expression ' - 'of the form\n' + 'argument. An *arg_name* is treated as a number if a call ' + 'to\n' + '"str.isdecimal()" on the string would return true. If the ' + 'numerical\n' + 'arg_names in a format string are 0, 1, 2, … in sequence, ' + 'they can all\n' + 'be omitted (not just some) and the numbers 0, 1, 2, … will ' + 'be\n' + 'automatically inserted in that order. Because *arg_name* is ' + 'not quote-\n' + 'delimited, it is not possible to specify arbitrary ' + 'dictionary keys\n' + '(e.g., the strings "\'10\'" or "\':-]\'") within a format ' + 'string. The\n' + '*arg_name* can be followed by any number of index or ' + 'attribute\n' + 'expressions. An expression of the form "\'.name\'" selects ' + 'the named\n' + 'attribute using "getattr()", while an expression of the ' + 'form\n' '"\'[index]\'" does an index lookup using "__getitem__()".\n' '\n' 'Changed in version 3.1: The positional argument specifiers ' @@ -13252,7 +13258,7 @@ 'are\n' 'most of the built-in objects considered false:\n' '\n' - '* constants defined to be false: "None" and "False".\n' + '* constants defined to be false: "None" and "False"\n' '\n' '* zero of any numeric type: "0", "0.0", "0j", "Decimal(0)",\n' ' "Fraction(0, 1)"\n' diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index 5709acb6267238..74bda2f103083c 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -114,7 +114,6 @@ def __init__(self, state, data=None): self.width = None def dump(self, level=0): - nl = True seqtypes = (tuple, list) for op, av in self.data: print(level*" " + str(op), end='') @@ -136,6 +135,9 @@ def dump(self, level=0): if item_no: print(level*" " + "ELSE") item_no.dump(level+1) + elif isinstance(av, SubPattern): + print() + av.dump(level+1) elif isinstance(av, seqtypes): nl = False for a in av: diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 767f049b0d717a..c13b049645ac90 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -13,6 +13,7 @@ import os import gc import errno +import functools import signal import array import socket @@ -31,6 +32,7 @@ from test.support import hashlib_helper from test.support import import_helper from test.support import os_helper +from test.support import script_helper from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper @@ -171,6 +173,59 @@ def check_enough_semaphores(): "to run the test (required: %d)." % nsems_min) +def only_run_in_spawn_testsuite(reason): + """Returns a decorator: raises SkipTest when SM != spawn at test time. + + This can be useful to save overall Python test suite execution time. + "spawn" is the universal mode available on all platforms so this limits the + decorated test to only execute within test_multiprocessing_spawn. + + This would not be necessary if we refactored our test suite to split things + into other test files when they are not start method specific to be rerun + under all start methods. + """ + + def decorator(test_item): + + @functools.wraps(test_item) + def spawn_check_wrapper(*args, **kwargs): + if (start_method := multiprocessing.get_start_method()) != "spawn": + raise unittest.SkipTest(f"{start_method=}, not 'spawn'; {reason}") + return test_item(*args, **kwargs) + + return spawn_check_wrapper + + return decorator + + +class TestInternalDecorators(unittest.TestCase): + """Logic within a test suite that could errantly skip tests? Test it!""" + + @unittest.skipIf(sys.platform == "win32", "test requires that fork exists.") + def test_only_run_in_spawn_testsuite(self): + if multiprocessing.get_start_method() != "spawn": + raise unittest.SkipTest("only run in test_multiprocessing_spawn.") + + try: + @only_run_in_spawn_testsuite("testing this decorator") + def return_four_if_spawn(): + return 4 + except Exception as err: + self.fail(f"expected decorated `def` not to raise; caught {err}") + + orig_start_method = multiprocessing.get_start_method(allow_none=True) + try: + multiprocessing.set_start_method("spawn", force=True) + self.assertEqual(return_four_if_spawn(), 4) + multiprocessing.set_start_method("fork", force=True) + with self.assertRaises(unittest.SkipTest) as ctx: + return_four_if_spawn() + self.assertIn("testing this decorator", str(ctx.exception)) + self.assertIn("start_method=", str(ctx.exception)) + finally: + multiprocessing.set_start_method(orig_start_method, force=True) + + # # Creates a wrapper for a function which records the time it takes to finish # @@ -5815,6 +5870,7 @@ def test_namespace(self): class TestNamedResource(unittest.TestCase): + @only_run_in_spawn_testsuite("spawn specific test.") def test_global_named_resource_spawn(self): # # gh-90549: Check that global named resources in main module @@ -5825,22 +5881,18 @@ def test_global_named_resource_spawn(self): with open(testfn, 'w', encoding='utf-8') as f: f.write(textwrap.dedent('''\ import multiprocessing as mp - ctx = mp.get_context('spawn') - global_resource = ctx.Semaphore() - def submain(): pass - if __name__ == '__main__': p = ctx.Process(target=submain) p.start() p.join() ''')) - rc, out, err = test.support.script_helper.assert_python_ok(testfn) + rc, out, err = script_helper.assert_python_ok(testfn) # on error, err = 'UserWarning: resource_tracker: There appear to # be 1 leaked semaphore objects to clean up at shutdown' - self.assertEqual(err, b'') + self.assertFalse(err, msg=err.decode('utf-8')) class MiscTestCase(unittest.TestCase): @@ -5849,6 +5901,24 @@ def test__all__(self): support.check__all__(self, multiprocessing, extra=multiprocessing.__all__, not_exported=['SUBDEBUG', 'SUBWARNING']) + @only_run_in_spawn_testsuite("avoids redundant testing.") + def test_spawn_sys_executable_none_allows_import(self): + # Regression test for a bug introduced in + # https://github.com/python/cpython/issues/90876 that caused an + # ImportError in multiprocessing when sys.executable was None. + # This can be true in embedded environments. + rc, out, err = script_helper.assert_python_ok( + "-c", + """if 1: + import sys + sys.executable = None + assert "multiprocessing" not in sys.modules, "already imported!" + import multiprocessing + import multiprocessing.spawn # This should not fail\n""", + ) + self.assertEqual(rc, 0) + self.assertFalse(err, msg=err.decode('utf-8')) + # # Mixins diff --git a/Lib/test/_testcppext.cpp b/Lib/test/_testcppext.cpp index 0e381a78c5ceed..82b471312dd2b9 100644 --- a/Lib/test/_testcppext.cpp +++ b/Lib/test/_testcppext.cpp @@ -86,7 +86,7 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) // gh-93442: Pass 0 as NULL for PyObject* Py_XINCREF(0); Py_XDECREF(0); -#if _cplusplus >= 201103 +#if __cplusplus >= 201103 // Test nullptr passed as PyObject* Py_XINCREF(nullptr); Py_XDECREF(nullptr); diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test.c similarity index 89% rename from Lib/test/clinic.test rename to Lib/test/clinic.test.c index 564205274edd73..1544599c29fa04 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test.c @@ -3,6 +3,10 @@ output preset block [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=3c81ac2402d06a8b]*/ +/*[clinic input] +class Test "TestObj *" "TestType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=fc7e50384d12b83f]*/ /*[clinic input] test_object_converter @@ -61,6 +65,58 @@ test_object_converter_impl(PyObject *module, PyObject *a, PyObject *b, /*[clinic end generated code: output=886f4f9b598726b6 input=005e6a8a711a869b]*/ +/*[clinic input] +cloned = test_object_converter +Check the clone feature. +[clinic start generated code]*/ + +PyDoc_STRVAR(cloned__doc__, +"cloned($module, a, b, c, d, /)\n" +"--\n" +"\n" +"Check the clone feature."); + +#define CLONED_METHODDEF \ + {"cloned", _PyCFunction_CAST(cloned), METH_FASTCALL, cloned__doc__}, + +static PyObject * +cloned_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyUnicode_Object *d); + +static PyObject * +cloned(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b; + PyObject *c; + PyUnicode_Object *d; + + if (!_PyArg_CheckPositional("cloned", nargs, 4, 4)) { + goto exit; + } + a = args[0]; + if (!PyUnicode_FSConverter(args[1], &b)) { + goto exit; + } + if (!PyUnicode_Check(args[2])) { + _PyArg_BadArgument("cloned", "argument 3", "str", args[2]); + goto exit; + } + c = args[2]; + d = (PyUnicode_Object *)args[3]; + return_value = cloned_impl(module, a, b, c, d); + +exit: + return return_value; +} + +static PyObject * +cloned_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyUnicode_Object *d) +/*[clinic end generated code: output=026b483e27c38065 input=0543614019d6fcc7]*/ + + /*[clinic input] test_object_converter_one_arg @@ -4271,3 +4327,500 @@ static PyObject * mangle2_impl(PyObject *module, PyObject *args, PyObject *kwargs, PyObject *return_value) /*[clinic end generated code: output=2ebb62aaefe7590a input=391766fee51bad7a]*/ + + +/*[clinic input] +Test.cls_with_param + cls: defining_class + / + a: int +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_cls_with_param__doc__, +"cls_with_param($self, /, a)\n" +"--\n" +"\n"); + +#define TEST_CLS_WITH_PARAM_METHODDEF \ + {"cls_with_param", _PyCFunction_CAST(Test_cls_with_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Test_cls_with_param__doc__}, + +static PyObject * +Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a); + +static PyObject * +Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "cls_with_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int a; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = _PyLong_AsInt(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = Test_cls_with_param_impl(self, cls, a); + +exit: + return return_value; +} + +static PyObject * +Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a) +/*[clinic end generated code: output=00218e7f583e6c81 input=af158077bd237ef9]*/ + + +/*[clinic input] +Test.__init__ +Empty init method. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test()\n" +"--\n" +"\n" +"Empty init method."); + +static int +Test___init___impl(TestObj *self); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoPositional("Test", args)) { + goto exit; + } + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + return_value = Test___init___impl((TestObj *)self); + +exit: + return return_value; +} + +static int +Test___init___impl(TestObj *self) +/*[clinic end generated code: output=f6a35c85bc5b408f input=4ea79fee54d0c3ff]*/ + + +/*[clinic input] +@classmethod +Test.__new__ +Empty new method. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test__doc__, +"Test()\n" +"--\n" +"\n" +"Empty new method."); + +static PyObject * +Test_impl(PyTypeObject *type); + +static PyObject * +Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = TestType; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("Test", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + return_value = Test_impl(type); + +exit: + return return_value; +} + +static PyObject * +Test_impl(PyTypeObject *type) +/*[clinic end generated code: output=68a117adc057940f input=6fe98a19f097907f]*/ + + +/*[clinic input] +Test.cls_no_params + cls: defining_class + / +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_cls_no_params__doc__, +"cls_no_params($self, /)\n" +"--\n" +"\n"); + +#define TEST_CLS_NO_PARAMS_METHODDEF \ + {"cls_no_params", _PyCFunction_CAST(Test_cls_no_params), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Test_cls_no_params__doc__}, + +static PyObject * +Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls); + +static PyObject * +Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs) { + PyErr_SetString(PyExc_TypeError, "cls_no_params() takes no arguments"); + return NULL; + } + return Test_cls_no_params_impl(self, cls); +} + +static PyObject * +Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls) +/*[clinic end generated code: output=cc8845f22cff3dcb input=e7e2e4e344e96a11]*/ + + +/*[clinic input] +Test.metho_not_default_return_converter -> int + a: object + / +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_metho_not_default_return_converter__doc__, +"metho_not_default_return_converter($self, a, /)\n" +"--\n" +"\n"); + +#define TEST_METHO_NOT_DEFAULT_RETURN_CONVERTER_METHODDEF \ + {"metho_not_default_return_converter", (PyCFunction)Test_metho_not_default_return_converter, METH_O, Test_metho_not_default_return_converter__doc__}, + +static int +Test_metho_not_default_return_converter_impl(TestObj *self, PyObject *a); + +static PyObject * +Test_metho_not_default_return_converter(TestObj *self, PyObject *a) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = Test_metho_not_default_return_converter_impl(self, a); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +static int +Test_metho_not_default_return_converter_impl(TestObj *self, PyObject *a) +/*[clinic end generated code: output=3350de11bd538007 input=428657129b521177]*/ + + +/*[clinic input] +Test.an_metho_arg_named_arg + arg: int + Name should be mangled to 'arg_' in generated output. + / +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_an_metho_arg_named_arg__doc__, +"an_metho_arg_named_arg($self, arg, /)\n" +"--\n" +"\n" +"\n" +"\n" +" arg\n" +" Name should be mangled to \'arg_\' in generated output."); + +#define TEST_AN_METHO_ARG_NAMED_ARG_METHODDEF \ + {"an_metho_arg_named_arg", (PyCFunction)Test_an_metho_arg_named_arg, METH_O, Test_an_metho_arg_named_arg__doc__}, + +static PyObject * +Test_an_metho_arg_named_arg_impl(TestObj *self, int arg); + +static PyObject * +Test_an_metho_arg_named_arg(TestObj *self, PyObject *arg_) +{ + PyObject *return_value = NULL; + int arg; + + arg = _PyLong_AsInt(arg_); + if (arg == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = Test_an_metho_arg_named_arg_impl(self, arg); + +exit: + return return_value; +} + +static PyObject * +Test_an_metho_arg_named_arg_impl(TestObj *self, int arg) +/*[clinic end generated code: output=7d590626642194ae input=2a53a57cf5624f95]*/ + + +/*[clinic input] +Test.__init__ + *args: object + / +Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test(*args)\n" +"--\n" +"\n" +"Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE."); + +static int +Test___init___impl(TestObj *self, PyObject *args); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + PyObject *__clinic_args = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_GetSlice(0, -1); + return_value = Test___init___impl((TestObj *)self, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +static int +Test___init___impl(TestObj *self, PyObject *args) +/*[clinic end generated code: output=0ed1009fe0dcf98d input=96c3ddc0cd38fc0c]*/ + + +/*[clinic input] +@classmethod +Test.__new__ + *args: object + / +Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test__doc__, +"Test(*args)\n" +"--\n" +"\n" +"Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE."); + +static PyObject * +Test_impl(PyTypeObject *type, PyObject *args); + +static PyObject * +Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = TestType; + PyObject *__clinic_args = NULL; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_GetSlice(0, -1); + return_value = Test_impl(type, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +static PyObject * +Test_impl(PyTypeObject *type, PyObject *args) +/*[clinic end generated code: output=8b219f6633e2a2e9 input=26a672e2e9750120]*/ + + +/*[clinic input] +Test.__init__ + a: object +Init method with positional or keyword arguments. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test(a)\n" +"--\n" +"\n" +"Init method with positional or keyword arguments."); + +static int +Test___init___impl(TestObj *self, PyObject *a); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "Test", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + return_value = Test___init___impl((TestObj *)self, a); + +exit: + return return_value; +} + +static int +Test___init___impl(TestObj *self, PyObject *a) +/*[clinic end generated code: output=0b9ca79638ab3ecb input=a8f9222a6ab35c59]*/ + + +/*[clinic input] +@classmethod +Test.class_method +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_class_method__doc__, +"class_method($type, /)\n" +"--\n" +"\n"); + +#define TEST_CLASS_METHOD_METHODDEF \ + {"class_method", (PyCFunction)Test_class_method, METH_NOARGS|METH_CLASS, Test_class_method__doc__}, + +static PyObject * +Test_class_method_impl(PyTypeObject *type); + +static PyObject * +Test_class_method(PyTypeObject *type, PyObject *Py_UNUSED(ignored)) +{ + return Test_class_method_impl(type); +} + +static PyObject * +Test_class_method_impl(PyTypeObject *type) +/*[clinic end generated code: output=47fb7ecca1abcaaa input=43bc4a0494547b80]*/ + + +/*[clinic input] +@staticmethod +Test.static_method +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_static_method__doc__, +"static_method()\n" +"--\n" +"\n"); + +#define TEST_STATIC_METHOD_METHODDEF \ + {"static_method", (PyCFunction)Test_static_method, METH_NOARGS|METH_STATIC, Test_static_method__doc__}, + +static PyObject * +Test_static_method_impl(); + +static PyObject * +Test_static_method(void *null, PyObject *Py_UNUSED(ignored)) +{ + return Test_static_method_impl(); +} + +static PyObject * +Test_static_method_impl() +/*[clinic end generated code: output=82524a63025cf7ab input=dae892fac55ae72b]*/ + + +/*[clinic input] +@coexist +Test.meth_coexist +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_meth_coexist__doc__, +"meth_coexist($self, /)\n" +"--\n" +"\n"); + +#define TEST_METH_COEXIST_METHODDEF \ + {"meth_coexist", (PyCFunction)Test_meth_coexist, METH_NOARGS|METH_COEXIST, Test_meth_coexist__doc__}, + +static PyObject * +Test_meth_coexist_impl(TestObj *self); + +static PyObject * +Test_meth_coexist(TestObj *self, PyObject *Py_UNUSED(ignored)) +{ + return Test_meth_coexist_impl(self); +} + +static PyObject * +Test_meth_coexist_impl(TestObj *self) +/*[clinic end generated code: output=808a293d0cd27439 input=2a1d75b5e6fec6dd]*/ diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 3c3509d0303371..9001ca33b6166a 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -526,6 +526,26 @@ def display_header(self): print("== CPU count:", cpu_count) print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) + asan = support.check_sanitizer(address=True) + msan = support.check_sanitizer(memory=True) + ubsan = support.check_sanitizer(ub=True) + # This makes it easier to remember what to set in your local + # environment when trying to reproduce a sanitizer failure. + if asan or msan or ubsan: + names = [n for n in (asan and "address", + msan and "memory", + ubsan and "undefined behavior") + if n] + print(f"== sanitizers: {', '.join(names)}") + a_opts = os.environ.get("ASAN_OPTIONS") + if asan and a_opts is not None: + print(f"== ASAN_OPTIONS={a_opts}") + m_opts = os.environ.get("ASAN_OPTIONS") + if msan and m_opts is not None: + print(f"== MSAN_OPTIONS={m_opts}") + ub_opts = os.environ.get("UBSAN_OPTIONS") + if ubsan and ub_opts is not None: + print(f"== UBSAN_OPTIONS={ub_opts}") def no_tests_run(self): return not any((self.good, self.bad, self.skipped, self.interrupted, diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index a12fcb46e0fd0b..62e6c6df36518c 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -277,6 +277,7 @@ def _runtest(self, test_name: str) -> MultiprocessResult: encoding = locale.getencoding() else: encoding = sys.stdout.encoding + # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. with tempfile.TemporaryFile('w+', encoding=encoding) as stdout_fh: @@ -298,7 +299,14 @@ def _runtest(self, test_name: str) -> MultiprocessResult: retcode = self._run_process(test_name, None, stdout_fh) tmp_files = () stdout_fh.seek(0) - stdout = stdout_fh.read().strip() + + try: + stdout = stdout_fh.read().strip() + except Exception as exc: + # gh-101634: Catch UnicodeDecodeError if stdout cannot be + # decoded from encoding + err_msg = f"Cannot read process stdout: {exc}" + return self.mp_result_error(ChildError(test_name), '', err_msg) if retcode is None: return self.mp_result_error(Timeout(test_name), stdout) @@ -481,6 +489,8 @@ def _process_result(self, item: QueueOutput) -> bool: # Thread got an exception format_exc = item[1] print_warning(f"regrtest worker thread failed: {format_exc}") + result = ChildError("") + self.regrtest.accumulate_result(result) return True self.test_index += 1 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d555c53fee50a2..3b332f49951f0c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -414,7 +414,7 @@ def check_sanitizer(*, address=False, memory=False, ub=False): ) address_sanitizer = ( '-fsanitize=address' in _cflags or - '--with-memory-sanitizer' in _config_args + '--with-address-sanitizer' in _config_args ) ub_sanitizer = ( '-fsanitize=undefined' in _cflags or @@ -1813,13 +1813,16 @@ def run_in_subinterp(code): return _testcapi.run_in_subinterp(code) -def run_in_subinterp_with_config(code, **config): +def run_in_subinterp_with_config(code, *, own_gil=None, **config): """ Run code in a subinterpreter. Raise unittest.SkipTest if the tracemalloc module is enabled. """ _check_tracemalloc() import _testcapi + if own_gil is not None: + assert 'gil' not in config, (own_gil, config) + config['gil'] = 2 if own_gil else 1 return _testcapi.run_in_subinterp_with_config(code, **config) @@ -1865,15 +1868,16 @@ def missing_compiler_executable(cmd_names=[]): missing. """ - # TODO (PEP 632): alternate check without using distutils - from distutils import ccompiler, sysconfig, spawn, errors + from setuptools._distutils import ccompiler, sysconfig, spawn + from setuptools import errors + compiler = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) if compiler.compiler_type == "msvc": # MSVC has no executables, so check whether initialization succeeds try: compiler.initialize() - except errors.DistutilsPlatformError: + except errors.PlatformError: return "msvc" for name in compiler.executables: if cmd_names and name not in cmd_names: @@ -2270,6 +2274,62 @@ def requires_venv_with_pip(): return unittest.skipUnless(ctypes, 'venv: pip requires ctypes') +@functools.cache +def _findwheel(pkgname): + """Try to find a wheel with the package specified as pkgname. + + If set, the wheels are searched for in WHEEL_PKG_DIR (see ensurepip). + Otherwise, they are searched for in the test directory. + """ + wheel_dir = sysconfig.get_config_var('WHEEL_PKG_DIR') or TEST_HOME_DIR + filenames = os.listdir(wheel_dir) + filenames = sorted(filenames, reverse=True) # approximate "newest" first + for filename in filenames: + # filename is like 'setuptools-67.6.1-py3-none-any.whl' + if not filename.endswith(".whl"): + continue + prefix = pkgname + '-' + if filename.startswith(prefix): + return os.path.join(wheel_dir, filename) + raise FileNotFoundError(f"No wheel for {pkgname} found in {wheel_dir}") + + +# Context manager that creates a virtual environment, install setuptools and wheel in it +# and returns the path to the venv directory and the path to the python executable +@contextlib.contextmanager +def setup_venv_with_pip_setuptools_wheel(venv_dir): + import subprocess + from .os_helper import temp_cwd + + with temp_cwd() as temp_dir: + # Create virtual environment to get setuptools + cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] + if verbose: + print() + print('Run:', ' '.join(cmd)) + subprocess.run(cmd, check=True) + + venv = os.path.join(temp_dir, venv_dir) + + # Get the Python executable of the venv + python_exe = os.path.basename(sys.executable) + if sys.platform == 'win32': + python = os.path.join(venv, 'Scripts', python_exe) + else: + python = os.path.join(venv, 'bin', python_exe) + + cmd = [python, '-X', 'dev', + '-m', 'pip', 'install', + _findwheel('setuptools'), + _findwheel('wheel')] + if verbose: + print() + print('Run:', ' '.join(cmd)) + subprocess.run(cmd, check=True) + + yield python + + # True if Python is built with the Py_DEBUG macro defined: if # Python is built in debug mode (./configure --with-pydebug). Py_DEBUG = hasattr(sys, 'gettotalrefcount') diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index eeff3abe0324e5..5c484d1170d1d9 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -5,7 +5,7 @@ import _xxinterpchannels as _channels # aliases: -from _xxsubinterpreters import is_shareable +from _xxsubinterpreters import is_shareable, RunFailedError from _xxinterpchannels import ( ChannelError, ChannelNotFoundError, ChannelEmptyError, ) diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index b9973c8bf5c914..7f16050f32b9d1 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -115,7 +115,11 @@ def join_thread(thread, timeout=None): @contextlib.contextmanager def start_threads(threads, unlock=None): - import faulthandler + try: + import faulthandler + except ImportError: + # It isn't supported on subinterpreters yet. + faulthandler = None threads = list(threads) started = [] try: @@ -147,7 +151,8 @@ def start_threads(threads, unlock=None): finally: started = [t for t in started if t.is_alive()] if started: - faulthandler.dump_traceback(sys.stdout) + if faulthandler is not None: + faulthandler.dump_traceback(sys.stdout) raise AssertionError('Unable to join %d threads' % len(started)) diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 86f31a9acb4d55..5ce57cc209ea85 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -448,15 +448,16 @@ class S(metaclass=abc_ABCMeta): # Also check that issubclass() propagates exceptions raised by # __subclasses__. + class CustomError(Exception): ... exc_msg = "exception from __subclasses__" def raise_exc(): - raise Exception(exc_msg) + raise CustomError(exc_msg) class S(metaclass=abc_ABCMeta): __subclasses__ = raise_exc - with self.assertRaisesRegex(Exception, exc_msg): + with self.assertRaisesRegex(CustomError, exc_msg): issubclass(int, S) def test_subclasshook(self): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index ffd082ec11806a..a03fa4c7187b05 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3,6 +3,7 @@ import dis import enum import os +import re import sys import textwrap import types @@ -1110,6 +1111,32 @@ def test_null_bytes(self): msg="source code string cannot contain null bytes"): ast.parse("a\0b") + def assert_none_check(self, node: type[ast.AST], attr: str, source: str) -> None: + with self.subTest(f"{node.__name__}.{attr}"): + tree = ast.parse(source) + found = 0 + for child in ast.walk(tree): + if isinstance(child, node): + setattr(child, attr, None) + found += 1 + self.assertEqual(found, 1) + e = re.escape(f"field '{attr}' is required for {node.__name__}") + with self.assertRaisesRegex(ValueError, f"^{e}$"): + compile(tree, "", "exec") + + def test_none_checks(self) -> None: + tests = [ + (ast.alias, "name", "import spam as SPAM"), + (ast.arg, "arg", "def spam(SPAM): spam"), + (ast.comprehension, "target", "[spam for SPAM in spam]"), + (ast.comprehension, "iter", "[spam for spam in SPAM]"), + (ast.keyword, "value", "spam(**SPAM)"), + (ast.match_case, "pattern", "match spam:\n case SPAM: spam"), + (ast.withitem, "context_expr", "with SPAM: spam"), + ] + for node, attr, source in tests: + self.assert_none_check(node, attr, source) + class ASTHelpers_Test(unittest.TestCase): maxDiff = None diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index fe690934292a86..346888735ff70b 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -12,7 +12,7 @@ from asyncio import tasks from test.test_asyncio import utils as test_utils from test.test_asyncio.test_tasks import get_innermost_context -from test import support +from test.support.script_helper import assert_python_ok MOCK_ANY = mock.ANY @@ -228,6 +228,23 @@ class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase class CEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) + def test_issue105987(self): + code = """if 1: + from _asyncio import _swap_current_task + + class DummyTask: + pass + + class DummyLoop: + pass + + l = DummyLoop() + _swap_current_task(l, DummyTask()) + t = _swap_current_task(l, None) + """ + + _, out, err = assert_python_ok("-c", code) + self.assertFalse(err) class AsyncTaskCounter: def __init__(self, loop, *, task_class, eager): diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 94fc9d4436b717..9f00bd5e76195f 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4718,7 +4718,7 @@ def __buffer__(self, flags): with self.assertRaises(ValueError): c.buffer.tobytes() - def test_multiple_inheritance_buffer_last(self): + def test_multiple_inheritance_buffer_last_raising(self): class A: def __buffer__(self, flags): raise RuntimeError("should not be called") diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index e46726192aa05b..682c56979c6dfa 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -1,10 +1,95 @@ import unittest +import sys from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +NULL = None + class CAPITest(unittest.TestCase): + # TODO: Test the following functions: + # + # PyUnicode_BuildEncodingMap + # PyUnicode_FSConverter + # PyUnicode_FSDecoder + # PyUnicode_DecodeMBCS + # PyUnicode_DecodeMBCSStateful + # PyUnicode_DecodeCodePageStateful + # PyUnicode_AsMBCSString + # PyUnicode_EncodeCodePage + # PyUnicode_DecodeLocaleAndSize + # PyUnicode_DecodeLocale + # PyUnicode_EncodeLocale + # PyUnicode_DecodeFSDefault + # PyUnicode_DecodeFSDefaultAndSize + # PyUnicode_EncodeFSDefault + + def test_fromencodedobject(self): + """Test PyUnicode_FromEncodedObject()""" + fromencodedobject = _testcapi.unicode_fromencodedobject + + self.assertEqual(fromencodedobject(b'abc', NULL), 'abc') + self.assertEqual(fromencodedobject(b'abc', 'ascii'), 'abc') + b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80' + s = 'a\xa1\u4f60\U0001f600' + self.assertEqual(fromencodedobject(b, NULL), s) + self.assertEqual(fromencodedobject(b, 'utf-8'), s) + self.assertEqual(fromencodedobject(b, 'latin1'), b.decode('latin1')) + self.assertRaises(UnicodeDecodeError, fromencodedobject, b, 'ascii') + self.assertEqual(fromencodedobject(b, 'ascii', 'replace'), + 'a' + '\ufffd'*9) + self.assertEqual(fromencodedobject(bytearray(b), NULL), s) + self.assertEqual(fromencodedobject(bytearray(b), 'utf-8'), s) + self.assertRaises(LookupError, fromencodedobject, b'abc', 'foo') + self.assertRaises(LookupError, fromencodedobject, b, 'ascii', 'foo') + self.assertRaises(TypeError, fromencodedobject, 'abc', NULL) + self.assertRaises(TypeError, fromencodedobject, 'abc', 'ascii') + self.assertRaises(TypeError, fromencodedobject, [], NULL) + self.assertRaises(TypeError, fromencodedobject, [], 'ascii') + self.assertRaises(SystemError, fromencodedobject, NULL, NULL) + self.assertRaises(SystemError, fromencodedobject, NULL, 'ascii') + + def test_decode(self): + """Test PyUnicode_Decode()""" + decode = _testcapi.unicode_decode + + self.assertEqual(decode(b'[\xe2\x82\xac]', 'utf-8'), '[\u20ac]') + self.assertEqual(decode(b'[\xa4]', 'iso8859-15'), '[\u20ac]') + self.assertEqual(decode(b'[\xa4]', 'iso8859-15', 'strict'), '[\u20ac]') + self.assertRaises(UnicodeDecodeError, decode, b'[\xa4]', 'utf-8') + self.assertEqual(decode(b'[\xa4]', 'utf-8', 'replace'), '[\ufffd]') + + self.assertEqual(decode(b'[\xe2\x82\xac]', NULL), '[\u20ac]') + self.assertEqual(decode(b'[\xa4]', NULL, 'replace'), '[\ufffd]') + + self.assertRaises(LookupError, decode, b'\xa4', 'foo') + self.assertRaises(LookupError, decode, b'\xa4', 'utf-8', 'foo') + # TODO: Test PyUnicode_Decode() with NULL as data and + # negative size. + + def test_asencodedstring(self): + """Test PyUnicode_AsEncodedString()""" + asencodedstring = _testcapi.unicode_asencodedstring + + self.assertEqual(asencodedstring('abc', NULL), b'abc') + self.assertEqual(asencodedstring('abc', 'ascii'), b'abc') + s = 'a\xa1\u4f60\U0001f600' + b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80' + self.assertEqual(asencodedstring(s, NULL), b) + self.assertEqual(asencodedstring(s, 'utf-8'), b) + self.assertEqual(asencodedstring('\xa1\xa2', 'latin1'), b'\xa1\xa2') + self.assertRaises(UnicodeEncodeError, asencodedstring, '\xa1\xa2', 'ascii') + self.assertEqual(asencodedstring(s, 'ascii', 'replace'), b'a???') + + self.assertRaises(LookupError, asencodedstring, 'abc', 'foo') + self.assertRaises(LookupError, asencodedstring, s, 'ascii', 'foo') + self.assertRaises(TypeError, asencodedstring, b'abc', NULL) + self.assertRaises(TypeError, asencodedstring, b'abc', 'ascii') + self.assertRaises(TypeError, asencodedstring, [], NULL) + self.assertRaises(TypeError, asencodedstring, [], 'ascii') + # CRASHES asencodedstring(NULL, NULL) + # CRASHES asencodedstring(NULL, 'ascii') def test_decodeutf8(self): """Test PyUnicode_DecodeUTF8()""" @@ -49,6 +134,387 @@ def test_decodeutf8stateful(self): # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of # "consumed". + def test_asutf8string(self): + """Test PyUnicode_AsUTF8String()""" + asutf8string = _testcapi.unicode_asutf8string + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + self.assertEqual(asutf8string(s), s.encode('utf-8')) + + self.assertRaises(UnicodeEncodeError, asutf8string, '\ud8ff') + self.assertRaises(TypeError, asutf8string, b'abc') + self.assertRaises(TypeError, asutf8string, []) + # CRASHES asutf8string(NULL) + + def test_decodeutf16(self): + """Test PyUnicode_DecodeUTF16()""" + decodeutf16 = _testcapi.unicode_decodeutf16 + + naturalbyteorder = -1 if sys.byteorder == 'little' else 1 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-16') + self.assertEqual(decodeutf16(0, b), (naturalbyteorder, s)) + b = s.encode('utf-16le') + self.assertEqual(decodeutf16(-1, b), (-1, s)) + self.assertEqual(decodeutf16(0, b'\xff\xfe'+b), (-1, s)) + b = s.encode('utf-16be') + self.assertEqual(decodeutf16(1, b), (1, s)) + self.assertEqual(decodeutf16(0, b'\xfe\xff'+b), (1, s)) + + self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'a') + self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'a') + self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xff\xfea') + self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xfe\xffa') + + self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x00\xde') + self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xde\x00') + self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xde\xde') + self.assertEqual(decodeutf16(-1, b'\x00\xde', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf16(1, b'\xde\x00', 'replace'), (1, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xde\xde', 'replace'), (0, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd')) + + self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x3d\xd8') + self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xd8\x3d') + self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xd8\xd8') + self.assertEqual(decodeutf16(-1, b'\x3d\xd8', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf16(1, b'\xd8\x3d', 'replace'), (1, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xd8\xd8', 'replace'), (0, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xff\xfe\x3d\xd8', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xfe\xff\xd8\x3d', 'replace'), (1, '\ufffd')) + + self.assertRaises(LookupError, decodeutf16, -1, b'\x00\xde', 'foo') + self.assertRaises(LookupError, decodeutf16, 1, b'\xde\x00', 'foo') + self.assertRaises(LookupError, decodeutf16, 0, b'\xde\xde', 'foo') + # TODO: Test PyUnicode_DecodeUTF16() with NULL as data and + # negative size. + + def test_decodeutf16stateful(self): + """Test PyUnicode_DecodeUTF16Stateful()""" + decodeutf16stateful = _testcapi.unicode_decodeutf16stateful + + naturalbyteorder = -1 if sys.byteorder == 'little' else 1 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-16') + self.assertEqual(decodeutf16stateful(0, b), (naturalbyteorder, s, len(b))) + b = s.encode('utf-16le') + self.assertEqual(decodeutf16stateful(-1, b), (-1, s, len(b))) + self.assertEqual(decodeutf16stateful(0, b'\xff\xfe'+b), (-1, s, len(b)+2)) + b = s.encode('utf-16be') + self.assertEqual(decodeutf16stateful(1, b), (1, s, len(b))) + self.assertEqual(decodeutf16stateful(0, b'\xfe\xff'+b), (1, s, len(b)+2)) + + self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d'), (-1, 'a', 2)) + self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8'), (-1, 'a', 2)) + self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8\x00'), (-1, 'a', 2)) + self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8'), (1, 'a', 2)) + self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d'), (1, 'a', 2)) + self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d\xde'), (1, 'a', 2)) + self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x61\x00\x3d\xd8\x00'), (-1, 'a', 4)) + self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\x00\x61\xd8\x3d\xde'), (1, 'a', 4)) + + self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x00\xde') + self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xde\x00') + self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 0, b'\xde\xde') + self.assertEqual(decodeutf16stateful(-1, b'\x00\xde', 'replace'), (-1, '\ufffd', 2)) + self.assertEqual(decodeutf16stateful(1, b'\xde\x00', 'replace'), (1, '\ufffd', 2)) + self.assertEqual(decodeutf16stateful(0, b'\xde\xde', 'replace'), (0, '\ufffd', 2)) + self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd', 4)) + self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd', 4)) + + self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x3d\xd8\x61\x00') + self.assertEqual(decodeutf16stateful(-1, b'\x3d\xd8\x61\x00', 'replace'), (-1, '\ufffda', 4)) + self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xd8\x3d\x00\x61') + self.assertEqual(decodeutf16stateful(1, b'\xd8\x3d\x00\x61', 'replace'), (1, '\ufffda', 4)) + + self.assertRaises(LookupError, decodeutf16stateful, -1, b'\x00\xde', 'foo') + self.assertRaises(LookupError, decodeutf16stateful, 1, b'\xde\x00', 'foo') + self.assertRaises(LookupError, decodeutf16stateful, 0, b'\xde\xde', 'foo') + # TODO: Test PyUnicode_DecodeUTF16Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF16Stateful() with NULL as the address of + # "consumed". + + def test_asutf16string(self): + """Test PyUnicode_AsUTF16String()""" + asutf16string = _testcapi.unicode_asutf16string + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + self.assertEqual(asutf16string(s), s.encode('utf-16')) + + self.assertRaises(UnicodeEncodeError, asutf16string, '\ud8ff') + self.assertRaises(TypeError, asutf16string, b'abc') + self.assertRaises(TypeError, asutf16string, []) + # CRASHES asutf16string(NULL) + + def test_decodeutf32(self): + """Test PyUnicode_DecodeUTF8()""" + decodeutf32 = _testcapi.unicode_decodeutf32 + + naturalbyteorder = -1 if sys.byteorder == 'little' else 1 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-32') + self.assertEqual(decodeutf32(0, b), (naturalbyteorder, s)) + b = s.encode('utf-32le') + self.assertEqual(decodeutf32(-1, b), (-1, s)) + self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00'+b), (-1, s)) + b = s.encode('utf-32be') + self.assertEqual(decodeutf32(1, b), (1, s)) + self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff'+b), (1, s)) + + self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x61\x00\x00\x00\x00') + self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\x00\x61\x00') + self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00') + self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00') + + self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\xff\xff\xff\xff') + self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\xff\xff\xff\xff') + self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xff\xff\xff') + self.assertEqual(decodeutf32(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf32(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd')) + self.assertEqual(decodeutf32(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd')) + self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd')) + + self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x3d\xd8\x00\x00') + self.assertEqual(decodeutf32(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd')) + self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\xd8\x3d') + self.assertEqual(decodeutf32(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd')) + + self.assertRaises(LookupError, decodeutf32, -1, b'\xff\xff\xff\xff', 'foo') + self.assertRaises(LookupError, decodeutf32, 1, b'\xff\xff\xff\xff', 'foo') + self.assertRaises(LookupError, decodeutf32, 0, b'\xff\xff\xff\xff', 'foo') + # TODO: Test PyUnicode_DecodeUTF32() with NULL as data and + # negative size. + + def test_decodeutf32stateful(self): + """Test PyUnicode_DecodeUTF32Stateful()""" + decodeutf32stateful = _testcapi.unicode_decodeutf32stateful + + naturalbyteorder = -1 if sys.byteorder == 'little' else 1 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-32') + self.assertEqual(decodeutf32stateful(0, b), (naturalbyteorder, s, len(b))) + b = s.encode('utf-32le') + self.assertEqual(decodeutf32stateful(-1, b), (-1, s, len(b))) + self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, s, len(b)+4)) + b = s.encode('utf-32be') + self.assertEqual(decodeutf32stateful(1, b), (1, s, len(b))) + self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, s, len(b)+4)) + + self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00'), (-1, 'a', 4)) + self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6'), (-1, 'a', 4)) + self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 4)) + self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00'), (1, 'a', 4)) + self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01'), (1, 'a', 4)) + self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 4)) + self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 8)) + self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 8)) + + for b in b'\xff', b'\xff\xff', b'\xff\xff\xff': + self.assertEqual(decodeutf32stateful(-1, b), (-1, '', 0)) + self.assertEqual(decodeutf32stateful(1, b), (1, '', 0)) + self.assertEqual(decodeutf32stateful(0, b), (0, '', 0)) + self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, '', 4)) + self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, '', 4)) + self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\xff\xff\xff\xff') + self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\xff\xff\xff\xff') + self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 0, b'\xff\xff\xff\xff') + self.assertEqual(decodeutf32stateful(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 4)) + self.assertEqual(decodeutf32stateful(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 4)) + self.assertEqual(decodeutf32stateful(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd', 4)) + self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 8)) + self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 8)) + + self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\x3d\xd8\x00\x00') + self.assertEqual(decodeutf32stateful(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd', 4)) + self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\x00\x00\xd8\x3d') + self.assertEqual(decodeutf32stateful(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd', 4)) + + self.assertRaises(LookupError, decodeutf32stateful, -1, b'\xff\xff\xff\xff', 'foo') + self.assertRaises(LookupError, decodeutf32stateful, 1, b'\xff\xff\xff\xff', 'foo') + self.assertRaises(LookupError, decodeutf32stateful, 0, b'\xff\xff\xff\xff', 'foo') + # TODO: Test PyUnicode_DecodeUTF32Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF32Stateful() with NULL as the address of + # "consumed". + + def test_asutf32string(self): + """Test PyUnicode_AsUTF32String()""" + asutf32string = _testcapi.unicode_asutf32string + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + self.assertEqual(asutf32string(s), s.encode('utf-32')) + + self.assertRaises(UnicodeEncodeError, asutf32string, '\ud8ff') + self.assertRaises(TypeError, asutf32string, b'abc') + self.assertRaises(TypeError, asutf32string, []) + # CRASHES asutf32string(NULL) + + def test_decodelatin1(self): + """Test PyUnicode_DecodeLatin1()""" + decodelatin1 = _testcapi.unicode_decodelatin1 + + self.assertEqual(decodelatin1(b'abc'), 'abc') + self.assertEqual(decodelatin1(b'abc', 'strict'), 'abc') + self.assertEqual(decodelatin1(b'\xa1\xa2'), '\xa1\xa2') + self.assertEqual(decodelatin1(b'\xa1\xa2', 'strict'), '\xa1\xa2') + # TODO: Test PyUnicode_DecodeLatin1() with NULL as data and + # negative size. + + def test_aslatin1string(self): + """Test PyUnicode_AsLatin1String()""" + aslatin1string = _testcapi.unicode_aslatin1string + + self.assertEqual(aslatin1string('abc'), b'abc') + self.assertEqual(aslatin1string('\xa1\xa2'), b'\xa1\xa2') + + self.assertRaises(UnicodeEncodeError, aslatin1string, '\u4f60') + self.assertRaises(TypeError, aslatin1string, b'abc') + self.assertRaises(TypeError, aslatin1string, []) + # CRASHES aslatin1string(NULL) + + def test_decodeascii(self): + """Test PyUnicode_DecodeASCII()""" + decodeascii = _testcapi.unicode_decodeascii + + self.assertEqual(decodeascii(b'abc'), 'abc') + self.assertEqual(decodeascii(b'abc', 'strict'), 'abc') + + self.assertRaises(UnicodeDecodeError, decodeascii, b'\xff') + self.assertEqual(decodeascii(b'a\xff', 'replace'), 'a\ufffd') + self.assertEqual(decodeascii(b'a\xffb', 'replace'), 'a\ufffdb') + + self.assertRaises(LookupError, decodeascii, b'a\xff', 'foo') + # TODO: Test PyUnicode_DecodeASCII() with NULL as data and + # negative size. + + def test_asasciistring(self): + """Test PyUnicode_AsASCIIString()""" + asasciistring = _testcapi.unicode_asasciistring + + self.assertEqual(asasciistring('abc'), b'abc') + + self.assertRaises(UnicodeEncodeError, asasciistring, '\x80') + self.assertRaises(TypeError, asasciistring, b'abc') + self.assertRaises(TypeError, asasciistring, []) + # CRASHES asasciistring(NULL) + + def test_decodecharmap(self): + """Test PyUnicode_DecodeCharmap()""" + decodecharmap = _testcapi.unicode_decodecharmap + + self.assertEqual(decodecharmap(b'\3\0\7', {0: 'a', 3: 'b', 7: 'c'}), 'bac') + self.assertEqual(decodecharmap(b'\1\0\2', ['a', 'b', 'c']), 'bac') + self.assertEqual(decodecharmap(b'\1\0\2', 'abc'), 'bac') + self.assertEqual(decodecharmap(b'\1\0\2', ['\xa1', '\xa2', '\xa3']), '\xa2\xa1\xa3') + self.assertEqual(decodecharmap(b'\1\0\2', ['\u4f60', '\u597d', '\u4e16']), '\u597d\u4f60\u4e16') + self.assertEqual(decodecharmap(b'\1\0\2', ['\U0001f600', '\U0001f601', '\U0001f602']), '\U0001f601\U0001f600\U0001f602') + + self.assertEqual(decodecharmap(b'\1\0\2', [97, 98, 99]), 'bac') + self.assertEqual(decodecharmap(b'\1\0\2', ['', 'b', 'cd']), 'bcd') + + self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {}) + self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {0: None}) + self.assertEqual(decodecharmap(b'\1\0\2', [None, 'b', 'c'], 'replace'), 'b\ufffdc') + self.assertEqual(decodecharmap(b'\1\0\2\xff', NULL), '\1\0\2\xff') + self.assertRaises(TypeError, decodecharmap, b'\0', 42) + + # TODO: Test PyUnicode_DecodeCharmap() with NULL as data and + # negative size. + + def test_ascharmapstring(self): + """Test PyUnicode_AsCharmapString()""" + ascharmapstring = _testcapi.unicode_ascharmapstring + + self.assertEqual(ascharmapstring('abc', {97: 3, 98: 0, 99: 7}), b'\3\0\7') + self.assertEqual(ascharmapstring('\xa1\xa2\xa3', {0xa1: 3, 0xa2: 0, 0xa3: 7}), b'\3\0\7') + self.assertEqual(ascharmapstring('\u4f60\u597d\u4e16', {0x4f60: 3, 0x597d: 0, 0x4e16: 7}), b'\3\0\7') + self.assertEqual(ascharmapstring('\U0001f600\U0001f601\U0001f602', {0x1f600: 3, 0x1f601: 0, 0x1f602: 7}), b'\3\0\7') + self.assertEqual(ascharmapstring('abc', {97: 3, 98: b'', 99: b'spam'}), b'\3spam') + + self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {}) + self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {97: None}) + self.assertRaises(TypeError, ascharmapstring, b'a', {}) + self.assertRaises(TypeError, ascharmapstring, [], {}) + self.assertRaises(TypeError, ascharmapstring, 'a', NULL) + # CRASHES ascharmapstring(NULL, {}) + + def test_decodeunicodeescape(self): + """Test PyUnicode_DecodeUnicodeEscape()""" + decodeunicodeescape = _testcapi.unicode_decodeunicodeescape + + self.assertEqual(decodeunicodeescape(b'abc'), 'abc') + self.assertEqual(decodeunicodeescape(br'\t\n\r\x0b\x0c\x00\\'), '\t\n\r\v\f\0\\') + self.assertEqual(decodeunicodeescape(b'\t\n\r\x0b\x0c\x00'), '\t\n\r\v\f\0') + self.assertEqual(decodeunicodeescape(br'\xa1\xa2'), '\xa1\xa2') + self.assertEqual(decodeunicodeescape(b'\xa1\xa2'), '\xa1\xa2') + self.assertEqual(decodeunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d') + self.assertEqual(decodeunicodeescape(br'\U0001f600'), '\U0001f600') + with self.assertWarns(DeprecationWarning): + self.assertEqual(decodeunicodeescape(br'\z'), r'\z') + + for b in b'\\', br'\xa', br'\u4f6', br'\U0001f60': + self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b) + self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b, 'strict') + self.assertEqual(decodeunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd') + self.assertEqual(decodeunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy') + + self.assertRaises(LookupError, decodeunicodeescape, b'\\', 'foo') + # TODO: Test PyUnicode_DecodeUnicodeEscape() with NULL as data and + # negative size. + + def test_asunicodeescapestring(self): + """Test PyUnicode_AsUnicodeEscapeString()""" + asunicodeescapestring = _testcapi.unicode_asunicodeescapestring + + self.assertEqual(asunicodeescapestring('abc'), b'abc') + self.assertEqual(asunicodeescapestring('\t\n\r\v\f\0\\'), br'\t\n\r\x0b\x0c\x00\\') + self.assertEqual(asunicodeescapestring('\xa1\xa2'), br'\xa1\xa2') + self.assertEqual(asunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d') + self.assertEqual(asunicodeescapestring('\U0001f600'), br'\U0001f600') + + self.assertRaises(TypeError, asunicodeescapestring, b'abc') + self.assertRaises(TypeError, asunicodeescapestring, []) + # CRASHES asunicodeescapestring(NULL) + + def test_decoderawunicodeescape(self): + """Test PyUnicode_DecodeRawUnicodeEscape()""" + decoderawunicodeescape = _testcapi.unicode_decoderawunicodeescape + + self.assertEqual(decoderawunicodeescape(b'abc'), 'abc') + self.assertEqual(decoderawunicodeescape(b'\t\n\r\v\f\0\\'), '\t\n\r\v\f\0\\') + self.assertEqual(decoderawunicodeescape(b'\xa1\xa2'), '\xa1\xa2') + self.assertEqual(decoderawunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d') + self.assertEqual(decoderawunicodeescape(br'\U0001f600'), '\U0001f600') + self.assertEqual(decoderawunicodeescape(br'\xa1\xa2'), r'\xa1\xa2') + self.assertEqual(decoderawunicodeescape(br'\z'), r'\z') + + for b in br'\u4f6', br'\U0001f60': + self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b) + self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b, 'strict') + self.assertEqual(decoderawunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd') + self.assertEqual(decoderawunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy') + + self.assertRaises(LookupError, decoderawunicodeescape, br'\U0001f60', 'foo') + # TODO: Test PyUnicode_DecodeRawUnicodeEscape() with NULL as data and + # negative size. + + def test_asrawunicodeescapestring(self): + """Test PyUnicode_AsRawUnicodeEscapeString()""" + asrawunicodeescapestring = _testcapi.unicode_asrawunicodeescapestring + + self.assertEqual(asrawunicodeescapestring('abc'), b'abc') + self.assertEqual(asrawunicodeescapestring('\t\n\r\v\f\0\\'), b'\t\n\r\v\f\0\\') + self.assertEqual(asrawunicodeescapestring('\xa1\xa2'), b'\xa1\xa2') + self.assertEqual(asrawunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d') + self.assertEqual(asrawunicodeescapestring('\U0001f600'), br'\U0001f600') + + self.assertRaises(TypeError, asrawunicodeescapestring, b'abc') + self.assertRaises(TypeError, asrawunicodeescapestring, []) + # CRASHES asrawunicodeescapestring(NULL) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index e1b55cffe8ff52..61947cb7945193 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2,18 +2,21 @@ # these are all functions _testcapi exports whose name begins with 'test_'. import _thread -from collections import OrderedDict +from collections import OrderedDict, deque import contextlib import importlib.machinery import importlib.util +import json import os import pickle +import queue import random import subprocess import sys import textwrap import threading import time +import types import unittest import warnings import weakref @@ -37,6 +40,10 @@ import _testsinglephase except ImportError: _testsinglephase = None +try: + import _xxsubinterpreters as _interpreters +except ModuleNotFoundError: + _interpreters = None # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') @@ -48,6 +55,12 @@ def decode_stderr(err): return err.decode('utf-8', 'replace').replace('\r', '') +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(_interpreters is None, + 'subinterpreters required')(meth) + + def testfunction(self): """some doc""" return self @@ -672,31 +685,60 @@ def test_heaptype_with_setattro(self): self.assertEqual(obj.pvalue, 0) def test_heaptype_with_custom_metaclass(self): - self.assertTrue(issubclass(_testcapi.HeapCTypeMetaclass, type)) - self.assertTrue(issubclass(_testcapi.HeapCTypeMetaclassCustomNew, type)) + metaclass = _testcapi.HeapCTypeMetaclass + self.assertTrue(issubclass(metaclass, type)) + + # Class creation from C + t = _testcapi.pytype_fromspec_meta(metaclass) + self.assertIsInstance(t, type) + self.assertEqual(t.__name__, "HeapCTypeViaMetaclass") + self.assertIs(type(t), metaclass) + + # Class creation from Python + t = metaclass("PyClassViaMetaclass", (), {}) + self.assertIsInstance(t, type) + self.assertEqual(t.__name__, "PyClassViaMetaclass") + + def test_heaptype_with_custom_metaclass_null_new(self): + metaclass = _testcapi.HeapCTypeMetaclassNullNew + + self.assertTrue(issubclass(metaclass, type)) - t = _testcapi.pytype_fromspec_meta(_testcapi.HeapCTypeMetaclass) + # Class creation from C + t = _testcapi.pytype_fromspec_meta(metaclass) self.assertIsInstance(t, type) self.assertEqual(t.__name__, "HeapCTypeViaMetaclass") - self.assertIs(type(t), _testcapi.HeapCTypeMetaclass) + self.assertIs(type(t), metaclass) + + # Class creation from Python + with self.assertRaisesRegex(TypeError, "cannot create .* instances"): + metaclass("PyClassViaMetaclass", (), {}) + + def test_heaptype_with_custom_metaclass_custom_new(self): + metaclass = _testcapi.HeapCTypeMetaclassCustomNew + + self.assertTrue(issubclass(_testcapi.HeapCTypeMetaclassCustomNew, type)) msg = "Metaclasses with custom tp_new are not supported." with self.assertRaisesRegex(TypeError, msg): - t = _testcapi.pytype_fromspec_meta(_testcapi.HeapCTypeMetaclassCustomNew) + t = _testcapi.pytype_fromspec_meta(metaclass) def test_heaptype_with_custom_metaclass_deprecation(self): + metaclass = _testcapi.HeapCTypeMetaclassCustomNew + # gh-103968: a metaclass with custom tp_new is deprecated, but still # allowed for functions that existed in 3.11 # (PyType_FromSpecWithBases is used here). - class Base(metaclass=_testcapi.HeapCTypeMetaclassCustomNew): + class Base(metaclass=metaclass): pass + # Class creation from C with warnings_helper.check_warnings( ('.*custom tp_new.*in Python 3.14.*', DeprecationWarning), ): sub = _testcapi.make_type_with_base(Base) self.assertTrue(issubclass(sub, Base)) - self.assertIsInstance(sub, _testcapi.HeapCTypeMetaclassCustomNew) + self.assertIsInstance(sub, metaclass) def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): @@ -1231,6 +1273,10 @@ def test_pyobject_getitemdata_error(self): class TestPendingCalls(unittest.TestCase): + # See the comment in ceval.c (at the "handle_eval_breaker" label) + # about when pending calls get run. This is especially relevant + # here for creating deterministic tests. + def pendingcalls_submit(self, l, n): def callback(): #this function can be interrupted by thread switching so let's @@ -1313,6 +1359,390 @@ def genf(): yield gen = genf() self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) + class PendingTask(types.SimpleNamespace): + + _add_pending = _testinternalcapi.pending_threadfunc + + def __init__(self, req, taskid=None, notify_done=None): + self.id = taskid + self.req = req + self.notify_done = notify_done + + self.creator_tid = threading.get_ident() + self.requester_tid = None + self.runner_tid = None + self.result = None + + def run(self): + assert self.result is None + self.runner_tid = threading.get_ident() + self._run() + if self.notify_done is not None: + self.notify_done() + + def _run(self): + self.result = self.req + + def run_in_pending_call(self, worker_tids): + assert self._add_pending is _testinternalcapi.pending_threadfunc + self.requester_tid = threading.get_ident() + def callback(): + assert self.result is None + # It can be tricky to control which thread handles + # the eval breaker, so we take a naive approach to + # make sure. + if threading.get_ident() not in worker_tids: + self._add_pending(callback, ensure_added=True) + return + self.run() + self._add_pending(callback, ensure_added=True) + + def create_thread(self, worker_tids): + return threading.Thread( + target=self.run_in_pending_call, + args=(worker_tids,), + ) + + def wait_for_result(self): + while self.result is None: + time.sleep(0.01) + + @threading_helper.requires_working_threading() + def test_subthreads_can_handle_pending_calls(self): + payload = 'Spam spam spam spam. Lovely spam! Wonderful spam!' + + task = self.PendingTask(payload) + def do_the_work(): + tid = threading.get_ident() + t = task.create_thread({tid}) + with threading_helper.start_threads([t]): + task.wait_for_result() + t = threading.Thread(target=do_the_work) + with threading_helper.start_threads([t]): + pass + + self.assertEqual(task.result, payload) + + @threading_helper.requires_working_threading() + def test_many_subthreads_can_handle_pending_calls(self): + main_tid = threading.get_ident() + self.assertEqual(threading.main_thread().ident, main_tid) + + # We can't use queue.Queue since it isn't reentrant relative + # to pending calls. + _queue = deque() + _active = deque() + _done_lock = threading.Lock() + def queue_put(task): + _queue.append(task) + _active.append(True) + def queue_get(): + try: + task = _queue.popleft() + except IndexError: + raise queue.Empty + return task + def queue_task_done(): + _active.pop() + if not _active: + try: + _done_lock.release() + except RuntimeError: + assert not _done_lock.locked() + def queue_empty(): + return not _queue + def queue_join(): + _done_lock.acquire() + _done_lock.release() + + tasks = [] + for i in range(20): + task = self.PendingTask( + req=f'request {i}', + taskid=i, + notify_done=queue_task_done, + ) + tasks.append(task) + queue_put(task) + # This will be released once all the tasks have finished. + _done_lock.acquire() + + def add_tasks(worker_tids): + while True: + if done: + return + try: + task = queue_get() + except queue.Empty: + break + task.run_in_pending_call(worker_tids) + + done = False + def run_tasks(): + while not queue_empty(): + if done: + return + time.sleep(0.01) + # Give the worker a chance to handle any remaining pending calls. + while not done: + time.sleep(0.01) + + # Start the workers and wait for them to finish. + worker_threads = [threading.Thread(target=run_tasks) + for _ in range(3)] + with threading_helper.start_threads(worker_threads): + try: + # Add a pending call for each task. + worker_tids = [t.ident for t in worker_threads] + threads = [threading.Thread(target=add_tasks, args=(worker_tids,)) + for _ in range(3)] + with threading_helper.start_threads(threads): + try: + pass + except BaseException: + done = True + raise # re-raise + # Wait for the pending calls to finish. + queue_join() + # Notify the workers that they can stop. + done = True + except BaseException: + done = True + raise # re-raise + runner_tids = [t.runner_tid for t in tasks] + + self.assertNotIn(main_tid, runner_tids) + for task in tasks: + with self.subTest(f'task {task.id}'): + self.assertNotEqual(task.requester_tid, main_tid) + self.assertNotEqual(task.requester_tid, task.runner_tid) + self.assertNotIn(task.requester_tid, runner_tids) + + @requires_subinterpreters + def test_isolated_subinterpreter(self): + # We exercise the most important permutations. + + # This test relies on pending calls getting called + # (eval breaker tripped) at each loop iteration + # and at each call. + + maxtext = 250 + main_interpid = 0 + interpid = _interpreters.create() + _interpreters.run_string(interpid, f"""if True: + import json + import os + import threading + import time + import _testinternalcapi + from test.support import threading_helper + """) + + def create_pipe(): + r, w = os.pipe() + self.addCleanup(lambda: os.close(r)) + self.addCleanup(lambda: os.close(w)) + return r, w + + with self.subTest('add in main, run in subinterpreter'): + r_ready, w_ready = create_pipe() + r_done, w_done= create_pipe() + timeout = time.time() + 30 # seconds + + def do_work(): + _interpreters.run_string(interpid, f"""if True: + # Wait until this interp has handled the pending call. + waiting = False + done = False + def wait(os_read=os.read): + global done, waiting + waiting = True + os_read({r_done}, 1) + done = True + t = threading.Thread(target=wait) + with threading_helper.start_threads([t]): + while not waiting: + pass + os.write({w_ready}, b'\\0') + # Loop to trigger the eval breaker. + while not done: + time.sleep(0.01) + if time.time() > {timeout}: + raise Exception('timed out!') + """) + t = threading.Thread(target=do_work) + with threading_helper.start_threads([t]): + os.read(r_ready, 1) + # Add the pending call and wait for it to finish. + actual = _testinternalcapi.pending_identify(interpid) + # Signal the subinterpreter to stop. + os.write(w_done, b'\0') + + self.assertEqual(actual, int(interpid)) + + with self.subTest('add in main, run in subinterpreter sub-thread'): + r_ready, w_ready = create_pipe() + r_done, w_done= create_pipe() + timeout = time.time() + 30 # seconds + + def do_work(): + _interpreters.run_string(interpid, f"""if True: + waiting = False + done = False + def subthread(): + while not waiting: + pass + os.write({w_ready}, b'\\0') + # Loop to trigger the eval breaker. + while not done: + time.sleep(0.01) + if time.time() > {timeout}: + raise Exception('timed out!') + t = threading.Thread(target=subthread) + with threading_helper.start_threads([t]): + # Wait until this interp has handled the pending call. + waiting = True + os.read({r_done}, 1) + done = True + """) + t = threading.Thread(target=do_work) + with threading_helper.start_threads([t]): + os.read(r_ready, 1) + # Add the pending call and wait for it to finish. + actual = _testinternalcapi.pending_identify(interpid) + # Signal the subinterpreter to stop. + os.write(w_done, b'\0') + + self.assertEqual(actual, int(interpid)) + + with self.subTest('add in subinterpreter, run in main'): + r_ready, w_ready = create_pipe() + r_done, w_done= create_pipe() + r_data, w_data= create_pipe() + timeout = time.time() + 30 # seconds + + def add_job(): + os.read(r_ready, 1) + _interpreters.run_string(interpid, f"""if True: + # Add the pending call and wait for it to finish. + actual = _testinternalcapi.pending_identify({main_interpid}) + # Signal the subinterpreter to stop. + os.write({w_done}, b'\\0') + os.write({w_data}, actual.to_bytes(1, 'little')) + """) + # Wait until this interp has handled the pending call. + waiting = False + done = False + def wait(os_read=os.read): + nonlocal done, waiting + waiting = True + os_read(r_done, 1) + done = True + t1 = threading.Thread(target=add_job) + t2 = threading.Thread(target=wait) + with threading_helper.start_threads([t1, t2]): + while not waiting: + pass + os.write(w_ready, b'\0') + # Loop to trigger the eval breaker. + while not done: + time.sleep(0.01) + if time.time() > timeout: + raise Exception('timed out!') + text = os.read(r_data, 1) + actual = int.from_bytes(text, 'little') + + self.assertEqual(actual, int(main_interpid)) + + with self.subTest('add in subinterpreter, run in sub-thread'): + r_ready, w_ready = create_pipe() + r_done, w_done= create_pipe() + r_data, w_data= create_pipe() + timeout = time.time() + 30 # seconds + + def add_job(): + os.read(r_ready, 1) + _interpreters.run_string(interpid, f"""if True: + # Add the pending call and wait for it to finish. + actual = _testinternalcapi.pending_identify({main_interpid}) + # Signal the subinterpreter to stop. + os.write({w_done}, b'\\0') + os.write({w_data}, actual.to_bytes(1, 'little')) + """) + # Wait until this interp has handled the pending call. + waiting = False + done = False + def wait(os_read=os.read): + nonlocal done, waiting + waiting = True + os_read(r_done, 1) + done = True + def subthread(): + while not waiting: + pass + os.write(w_ready, b'\0') + # Loop to trigger the eval breaker. + while not done: + time.sleep(0.01) + if time.time() > timeout: + raise Exception('timed out!') + t1 = threading.Thread(target=add_job) + t2 = threading.Thread(target=wait) + t3 = threading.Thread(target=subthread) + with threading_helper.start_threads([t1, t2, t3]): + pass + text = os.read(r_data, 1) + actual = int.from_bytes(text, 'little') + + self.assertEqual(actual, int(main_interpid)) + + # XXX We can't use the rest until gh-105716 is fixed. + return + + with self.subTest('add in subinterpreter, run in subinterpreter sub-thread'): + r_ready, w_ready = create_pipe() + r_done, w_done= create_pipe() + r_data, w_data= create_pipe() + timeout = time.time() + 30 # seconds + + def do_work(): + _interpreters.run_string(interpid, f"""if True: + waiting = False + done = False + def subthread(): + while not waiting: + pass + os.write({w_ready}, b'\\0') + # Loop to trigger the eval breaker. + while not done: + time.sleep(0.01) + if time.time() > {timeout}: + raise Exception('timed out!') + t = threading.Thread(target=subthread) + with threading_helper.start_threads([t]): + # Wait until this interp has handled the pending call. + waiting = True + os.read({r_done}, 1) + done = True + """) + t = threading.Thread(target=do_work) + #with threading_helper.start_threads([t]): + t.start() + if True: + os.read(r_ready, 1) + _interpreters.run_string(interpid, f"""if True: + # Add the pending call and wait for it to finish. + actual = _testinternalcapi.pending_identify({interpid}) + # Signal the subinterpreter to stop. + os.write({w_done}, b'\\0') + os.write({w_data}, actual.to_bytes(1, 'little')) + """) + t.join() + text = os.read(r_data, 1) + actual = int.from_bytes(text, 'little') + + self.assertEqual(actual, int(interpid)) + class SubinterpreterTest(unittest.TestCase): diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index bea7303805567f..b5744f7013d6ad 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -3,7 +3,8 @@ # Licensed to the PSF under a contributor agreement. from test import support, test_tools -from test.support import import_helper, os_helper +from test.support import os_helper +from textwrap import dedent from unittest import TestCase import collections import inspect @@ -17,6 +18,19 @@ from clinic import DSLParser +class _ParserBase(TestCase): + maxDiff = None + + def expect_parser_failure(self, parser, _input): + with support.captured_stdout() as stdout: + with self.assertRaises(SystemExit): + parser(_input) + return stdout.getvalue() + + def parse_function_should_fail(self, _input): + return self.expect_parser_failure(self.parse_function, _input) + + class FakeConverter: def __init__(self, name, args): self.name = name @@ -87,7 +101,15 @@ def directive(self, name, args): _module_and_class = clinic.Clinic._module_and_class -class ClinicWholeFileTest(TestCase): + +class ClinicWholeFileTest(_ParserBase): + def setUp(self): + self.clinic = clinic.Clinic(clinic.CLanguage(None), filename="test.c") + + def expect_failure(self, raw): + _input = dedent(raw).strip() + return self.expect_parser_failure(self.clinic.parse, _input) + def test_eol(self): # regression test: # clinic's block parser didn't recognize @@ -97,16 +119,86 @@ def test_eol(self): # so it would spit out an end line for you. # and since you really already had one, # the last line of the block got corrupted. - c = clinic.Clinic(clinic.CLanguage(None), filename="file") raw = "/*[clinic]\nfoo\n[clinic]*/" - cooked, _ = c.parse(raw) - lines = cooked.splitlines() - end_line = lines[2].rstrip() + cooked = self.clinic.parse(raw).splitlines() + end_line = cooked[2].rstrip() # this test is redundant, it's just here explicitly to catch # the regression test so we don't forget what it looked like self.assertNotEqual(end_line, "[clinic]*/[clinic]*/") self.assertEqual(end_line, "[clinic]*/") + def test_mangled_marker_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + /*[clinic end generated code: foo]*/ + """ + msg = ( + 'Error in file "test.c" on line 3:\n' + "Mangled Argument Clinic marker line: '/*[clinic end generated code: foo]*/'\n" + ) + out = self.expect_failure(raw) + self.assertEqual(out, msg) + + def test_checksum_mismatch(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + /*[clinic end generated code: output=0123456789abcdef input=fedcba9876543210]*/ + """ + msg = ( + 'Error in file "test.c" on line 3:\n' + 'Checksum mismatch!\n' + 'Expected: 0123456789abcdef\n' + 'Computed: da39a3ee5e6b4b0d\n' + ) + out = self.expect_failure(raw) + self.assertIn(msg, out) + + def test_garbage_after_stop_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/foobarfoobar! + """ + msg = ( + 'Error in file "test.c" on line 2:\n' + "Garbage after stop line: 'foobarfoobar!'\n" + ) + out = self.expect_failure(raw) + self.assertEqual(out, msg) + + def test_whitespace_before_stop_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + """ + msg = ( + 'Error in file "test.c" on line 2:\n' + "Whitespace is not allowed before the stop line: ' [clinic start generated code]*/'\n" + ) + out = self.expect_failure(raw) + self.assertEqual(out, msg) + + def test_parse_with_body_prefix(self): + clang = clinic.CLanguage(None) + clang.body_prefix = "//" + clang.start_line = "//[{dsl_name} start]" + clang.stop_line = "//[{dsl_name} stop]" + cl = clinic.Clinic(clang, filename="test.c") + raw = dedent(""" + //[clinic start] + //module test + //[clinic stop] + """).strip() + out = cl.parse(raw) + expected = dedent(""" + //[clinic start] + //module test + // + //[clinic stop] + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=65fab8adff58cf08]*/ + """).lstrip() # Note, lstrip() because of the newline + self.assertEqual(out, expected) class ClinicGroupPermuterTest(TestCase): @@ -171,43 +263,43 @@ def test_solo_newline(self): def test_no_substitution(self): self._test(""" abc - """, """ + """, """ abc - """) + """) def test_empty_substitution(self): self._test(""" abc {name} def - """, """ + """, """ abc def - """, name='') + """, name='') def test_single_line_substitution(self): self._test(""" abc {name} def - """, """ + """, """ abc GARGLE def - """, name='GARGLE') + """, name='GARGLE') def test_multiline_substitution(self): self._test(""" abc {name} def - """, """ + """, """ abc bingle bungle def - """, name='bingle\nbungle\n') + """, name='bingle\nbungle\n') class InertParser: def __init__(self, clinic): @@ -240,9 +332,9 @@ def round_trip(self, input): def test_round_trip_1(self): self.round_trip(""" - verbatim text here - lah dee dah -""") + verbatim text here + lah dee dah + """) def test_round_trip_2(self): self.round_trip(""" verbatim text here @@ -260,7 +352,7 @@ def _test_clinic(self, input, output): c = clinic.Clinic(language, filename="file") c.parsers['inert'] = InertParser(c) c.parsers['copy'] = CopyParser(c) - computed, _ = c.parse(input) + computed = c.parse(input) self.assertEqual(output, computed) def test_clinic_1(self): @@ -285,23 +377,39 @@ def test_clinic_1(self): """) -class ClinicParserTest(TestCase): +class ClinicParserTest(_ParserBase): + def checkDocstring(self, fn, expected): + self.assertTrue(hasattr(fn, "docstring")) + self.assertEqual(fn.docstring.strip(), + dedent(expected).strip()) + def test_trivial(self): parser = DSLParser(FakeClinic()) - block = clinic.Block("module os\nos.access") + block = clinic.Block(""" + module os + os.access + """) parser.parse(block) module, function = block.signatures self.assertEqual("access", function.name) self.assertEqual("os", module.name) def test_ignore_line(self): - block = self.parse("#\nmodule os\nos.access") + block = self.parse(dedent(""" + # + module os + os.access + """)) module, function = block.signatures self.assertEqual("access", function.name) self.assertEqual("os", module.name) def test_param(self): - function = self.parse_function("module os\nos.access\n path: int") + function = self.parse_function(""" + module os + os.access + path: int + """) self.assertEqual("access", function.name) self.assertEqual(2, len(function.parameters)) p = function.parameters['path'] @@ -309,236 +417,296 @@ def test_param(self): self.assertIsInstance(p.converter, clinic.int_converter) def test_param_default(self): - function = self.parse_function("module os\nos.access\n follow_symlinks: bool = True") + function = self.parse_function(""" + module os + os.access + follow_symlinks: bool = True + """) p = function.parameters['follow_symlinks'] self.assertEqual(True, p.default) def test_param_with_continuations(self): - function = self.parse_function("module os\nos.access\n follow_symlinks: \\\n bool \\\n =\\\n True") + function = self.parse_function(r""" + module os + os.access + follow_symlinks: \ + bool \ + = \ + True + """) p = function.parameters['follow_symlinks'] self.assertEqual(True, p.default) def test_param_default_expression(self): - function = self.parse_function("module os\nos.access\n follow_symlinks: int(c_default='MAXSIZE') = sys.maxsize") + function = self.parse_function(""" + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = sys.maxsize + """) p = function.parameters['follow_symlinks'] self.assertEqual(sys.maxsize, p.default) self.assertEqual("MAXSIZE", p.converter.c_default) - s = self.parse_function_should_fail("module os\nos.access\n follow_symlinks: int = sys.maxsize") - self.assertEqual(s, "Error on line 0:\nWhen you specify a named constant ('sys.maxsize') as your default value,\nyou MUST specify a valid c_default.\n") + expected_msg = ( + "Error on line 0:\n" + "When you specify a named constant ('sys.maxsize') as your default value,\n" + "you MUST specify a valid c_default.\n" + ) + out = self.parse_function_should_fail(""" + module os + os.access + follow_symlinks: int = sys.maxsize + """) + self.assertEqual(out, expected_msg) def test_param_no_docstring(self): function = self.parse_function(""" -module os -os.access - follow_symlinks: bool = True - something_else: str = ''""") + module os + os.access + follow_symlinks: bool = True + something_else: str = '' + """) p = function.parameters['follow_symlinks'] self.assertEqual(3, len(function.parameters)) - self.assertIsInstance(function.parameters['something_else'].converter, clinic.str_converter) + conv = function.parameters['something_else'].converter + self.assertIsInstance(conv, clinic.str_converter) def test_param_default_parameters_out_of_order(self): - s = self.parse_function_should_fail(""" -module os -os.access - follow_symlinks: bool = True - something_else: str""") - self.assertEqual(s, """Error on line 0: -Can't have a parameter without a default ('something_else') -after a parameter with a default! -""") + expected_msg = ( + "Error on line 0:\n" + "Can't have a parameter without a default ('something_else')\n" + "after a parameter with a default!\n" + ) + out = self.parse_function_should_fail(""" + module os + os.access + follow_symlinks: bool = True + something_else: str""") + self.assertEqual(out, expected_msg) def disabled_test_converter_arguments(self): - function = self.parse_function("module os\nos.access\n path: path_t(allow_fd=1)") + function = self.parse_function(""" + module os + os.access + path: path_t(allow_fd=1) + """) p = function.parameters['path'] self.assertEqual(1, p.converter.args['allow_fd']) def test_function_docstring(self): function = self.parse_function(""" -module os -os.stat as os_stat_fn + module os + os.stat as os_stat_fn - path: str - Path to be examined + path: str + Path to be examined -Perform a stat system call on the given path.""") - self.assertEqual(""" -stat($module, /, path) --- + Perform a stat system call on the given path. + """) + self.checkDocstring(function, """ + stat($module, /, path) + -- -Perform a stat system call on the given path. + Perform a stat system call on the given path. - path - Path to be examined -""".strip(), function.docstring) + path + Path to be examined + """) def test_explicit_parameters_in_docstring(self): - function = self.parse_function(""" -module foo -foo.bar - x: int - Documentation for x. - y: int + function = self.parse_function(dedent(""" + module foo + foo.bar + x: int + Documentation for x. + y: int -This is the documentation for foo. + This is the documentation for foo. -Okay, we're done here. -""") - self.assertEqual(""" -bar($module, /, x, y) --- + Okay, we're done here. + """)) + self.checkDocstring(function, """ + bar($module, /, x, y) + -- -This is the documentation for foo. + This is the documentation for foo. - x - Documentation for x. + x + Documentation for x. -Okay, we're done here. -""".strip(), function.docstring) + Okay, we're done here. + """) def test_parser_regression_special_character_in_parameter_column_of_docstring_first_line(self): - function = self.parse_function(""" -module os -os.stat - path: str -This/used to break Clinic! -""") - self.assertEqual("stat($module, /, path)\n--\n\nThis/used to break Clinic!", function.docstring) + function = self.parse_function(dedent(""" + module os + os.stat + path: str + This/used to break Clinic! + """)) + self.checkDocstring(function, """ + stat($module, /, path) + -- + + This/used to break Clinic! + """) def test_c_name(self): - function = self.parse_function("module os\nos.stat as os_stat_fn") + function = self.parse_function(""" + module os + os.stat as os_stat_fn + """) self.assertEqual("os_stat_fn", function.c_basename) def test_return_converter(self): - function = self.parse_function("module os\nos.stat -> int") + function = self.parse_function(""" + module os + os.stat -> int + """) self.assertIsInstance(function.return_converter, clinic.int_return_converter) def test_star(self): - function = self.parse_function("module os\nos.access\n *\n follow_symlinks: bool = True") + function = self.parse_function(""" + module os + os.access + * + follow_symlinks: bool = True + """) p = function.parameters['follow_symlinks'] self.assertEqual(inspect.Parameter.KEYWORD_ONLY, p.kind) self.assertEqual(0, p.group) def test_group(self): - function = self.parse_function("module window\nwindow.border\n [\n ls : int\n ]\n /\n") + function = self.parse_function(""" + module window + window.border + [ + ls: int + ] + / + """) p = function.parameters['ls'] self.assertEqual(1, p.group) def test_left_group(self): function = self.parse_function(""" -module curses -curses.addch - [ - y: int - Y-coordinate. - x: int - X-coordinate. - ] - ch: char - Character to add. - [ - attr: long - Attributes for the character. - ] - / -""") - for name, group in ( + module curses + curses.addch + [ + y: int + Y-coordinate. + x: int + X-coordinate. + ] + ch: char + Character to add. + [ + attr: long + Attributes for the character. + ] + / + """) + dataset = ( ('y', -1), ('x', -1), ('ch', 0), ('attr', 1), - ): - p = function.parameters[name] - self.assertEqual(p.group, group) - self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) - self.assertEqual(function.docstring.strip(), """ -addch([y, x,] ch, [attr]) - - - y - Y-coordinate. - x - X-coordinate. - ch - Character to add. - attr - Attributes for the character. - """.strip()) + ) + for name, group in dataset: + with self.subTest(name=name, group=group): + p = function.parameters[name] + self.assertEqual(p.group, group) + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + self.checkDocstring(function, """ + addch([y, x,] ch, [attr]) + + + y + Y-coordinate. + x + X-coordinate. + ch + Character to add. + attr + Attributes for the character. + """) def test_nested_groups(self): function = self.parse_function(""" -module curses -curses.imaginary - [ - [ - y1: int - Y-coordinate. - y2: int - Y-coordinate. - ] - x1: int - X-coordinate. - x2: int - X-coordinate. - ] - ch: char - Character to add. - [ - attr1: long - Attributes for the character. - attr2: long - Attributes for the character. - attr3: long - Attributes for the character. - [ - attr4: long - Attributes for the character. - attr5: long - Attributes for the character. - attr6: long - Attributes for the character. - ] - ] - / -""") - for name, group in ( + module curses + curses.imaginary + [ + [ + y1: int + Y-coordinate. + y2: int + Y-coordinate. + ] + x1: int + X-coordinate. + x2: int + X-coordinate. + ] + ch: char + Character to add. + [ + attr1: long + Attributes for the character. + attr2: long + Attributes for the character. + attr3: long + Attributes for the character. + [ + attr4: long + Attributes for the character. + attr5: long + Attributes for the character. + attr6: long + Attributes for the character. + ] + ] + / + """) + dataset = ( ('y1', -2), ('y2', -2), ('x1', -1), ('x2', -1), ('ch', 0), ('attr1', 1), ('attr2', 1), ('attr3', 1), ('attr4', 2), ('attr5', 2), ('attr6', 2), - ): - p = function.parameters[name] - self.assertEqual(p.group, group) - self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) - - self.assertEqual(function.docstring.strip(), """ -imaginary([[y1, y2,] x1, x2,] ch, [attr1, attr2, attr3, [attr4, attr5, - attr6]]) - - - y1 - Y-coordinate. - y2 - Y-coordinate. - x1 - X-coordinate. - x2 - X-coordinate. - ch - Character to add. - attr1 - Attributes for the character. - attr2 - Attributes for the character. - attr3 - Attributes for the character. - attr4 - Attributes for the character. - attr5 - Attributes for the character. - attr6 - Attributes for the character. - """.strip()) + ) + for name, group in dataset: + with self.subTest(name=name, group=group): + p = function.parameters[name] + self.assertEqual(p.group, group) + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + + self.checkDocstring(function, """ + imaginary([[y1, y2,] x1, x2,] ch, [attr1, attr2, attr3, [attr4, attr5, + attr6]]) + + + y1 + Y-coordinate. + y2 + Y-coordinate. + x1 + X-coordinate. + x2 + X-coordinate. + ch + Character to add. + attr1 + Attributes for the character. + attr2 + Attributes for the character. + attr3 + Attributes for the character. + attr4 + Attributes for the character. + attr5 + Attributes for the character. + attr6 + Attributes for the character. + """) def parse_function_should_fail(self, s): with support.captured_stdout() as stdout: @@ -547,218 +715,276 @@ def parse_function_should_fail(self, s): return stdout.getvalue() def test_disallowed_grouping__two_top_groups_on_left(self): - s = self.parse_function_should_fail(""" -module foo -foo.two_top_groups_on_left - [ - group1 : int - ] - [ - group2 : int - ] - param: int - """) - self.assertEqual(s, - ('Error on line 0:\n' - 'Function two_top_groups_on_left has an unsupported group configuration. (Unexpected state 2.b)\n')) + expected_msg = ( + 'Error on line 0:\n' + 'Function two_top_groups_on_left has an unsupported group ' + 'configuration. (Unexpected state 2.b)\n' + ) + out = self.parse_function_should_fail(""" + module foo + foo.two_top_groups_on_left + [ + group1 : int + ] + [ + group2 : int + ] + param: int + """) + self.assertEqual(out, expected_msg) def test_disallowed_grouping__two_top_groups_on_right(self): - self.parse_function_should_fail(""" -module foo -foo.two_top_groups_on_right - param: int - [ - group1 : int - ] - [ - group2 : int - ] - """) + out = self.parse_function_should_fail(""" + module foo + foo.two_top_groups_on_right + param: int + [ + group1 : int + ] + [ + group2 : int + ] + """) + msg = ( + "Function two_top_groups_on_right has an unsupported group " + "configuration. (Unexpected state 6.b)" + ) + self.assertIn(msg, out) def test_disallowed_grouping__parameter_after_group_on_right(self): - self.parse_function_should_fail(""" -module foo -foo.parameter_after_group_on_right - param: int - [ - [ - group1 : int - ] - group2 : int - ] - """) + out = self.parse_function_should_fail(""" + module foo + foo.parameter_after_group_on_right + param: int + [ + [ + group1 : int + ] + group2 : int + ] + """) + msg = ( + "Function parameter_after_group_on_right has an unsupported group " + "configuration. (Unexpected state 6.a)" + ) + self.assertIn(msg, out) def test_disallowed_grouping__group_after_parameter_on_left(self): - self.parse_function_should_fail(""" -module foo -foo.group_after_parameter_on_left - [ - group2 : int - [ - group1 : int - ] - ] - param: int - """) + out = self.parse_function_should_fail(""" + module foo + foo.group_after_parameter_on_left + [ + group2 : int + [ + group1 : int + ] + ] + param: int + """) + msg = ( + "Function group_after_parameter_on_left has an unsupported group " + "configuration. (Unexpected state 2.b)" + ) + self.assertIn(msg, out) def test_disallowed_grouping__empty_group_on_left(self): - self.parse_function_should_fail(""" -module foo -foo.empty_group - [ - [ - ] - group2 : int - ] - param: int - """) + out = self.parse_function_should_fail(""" + module foo + foo.empty_group + [ + [ + ] + group2 : int + ] + param: int + """) + msg = ( + "Function empty_group has an empty group.\n" + "All groups must contain at least one parameter." + ) + self.assertIn(msg, out) def test_disallowed_grouping__empty_group_on_right(self): - self.parse_function_should_fail(""" -module foo -foo.empty_group - param: int - [ - [ - ] - group2 : int - ] - """) + out = self.parse_function_should_fail(""" + module foo + foo.empty_group + param: int + [ + [ + ] + group2 : int + ] + """) + msg = ( + "Function empty_group has an empty group.\n" + "All groups must contain at least one parameter." + ) + self.assertIn(msg, out) + + def test_disallowed_grouping__no_matching_bracket(self): + out = self.parse_function_should_fail(""" + module foo + foo.empty_group + param: int + ] + group2: int + ] + """) + msg = "Function empty_group has a ] without a matching [." + self.assertIn(msg, out) def test_no_parameters(self): function = self.parse_function(""" -module foo -foo.bar + module foo + foo.bar -Docstring + Docstring -""") + """) self.assertEqual("bar($module, /)\n--\n\nDocstring", function.docstring) self.assertEqual(1, len(function.parameters)) # self! def test_init_with_no_parameters(self): function = self.parse_function(""" -module foo -class foo.Bar "unused" "notneeded" -foo.Bar.__init__ + module foo + class foo.Bar "unused" "notneeded" + foo.Bar.__init__ + + Docstring -Docstring + """, signatures_in_block=3, function_index=2) -""", signatures_in_block=3, function_index=2) # self is not in the signature self.assertEqual("Bar()\n--\n\nDocstring", function.docstring) # but it *is* a parameter self.assertEqual(1, len(function.parameters)) def test_illegal_module_line(self): - self.parse_function_should_fail(""" -module foo -foo.bar => int - / -""") + out = self.parse_function_should_fail(""" + module foo + foo.bar => int + / + """) + msg = "Illegal function name: foo.bar => int" + self.assertIn(msg, out) def test_illegal_c_basename(self): - self.parse_function_should_fail(""" -module foo -foo.bar as 935 - / -""") + out = self.parse_function_should_fail(""" + module foo + foo.bar as 935 + / + """) + msg = "Illegal C basename: 935" + self.assertIn(msg, out) def test_single_star(self): - self.parse_function_should_fail(""" -module foo -foo.bar - * - * -""") - - def test_parameters_required_after_star_without_initial_parameters_or_docstring(self): - self.parse_function_should_fail(""" -module foo -foo.bar - * -""") - - def test_parameters_required_after_star_without_initial_parameters_with_docstring(self): - self.parse_function_should_fail(""" -module foo -foo.bar - * -Docstring here. -""") - - def test_parameters_required_after_star_with_initial_parameters_without_docstring(self): - self.parse_function_should_fail(""" -module foo -foo.bar - this: int - * -""") + out = self.parse_function_should_fail(""" + module foo + foo.bar + * + * + """) + self.assertIn("Function bar uses '*' more than once.", out) - def test_parameters_required_after_star_with_initial_parameters_and_docstring(self): - self.parse_function_should_fail(""" -module foo -foo.bar - this: int - * -Docstring. -""") + def test_parameters_required_after_star(self): + dataset = ( + "module foo\nfoo.bar\n *", + "module foo\nfoo.bar\n *\nDocstring here.", + "module foo\nfoo.bar\n this: int\n *", + "module foo\nfoo.bar\n this: int\n *\nDocstring.", + ) + msg = "Function bar specifies '*' without any parameters afterwards." + for block in dataset: + with self.subTest(block=block): + out = self.parse_function_should_fail(block) + self.assertIn(msg, out) def test_single_slash(self): - self.parse_function_should_fail(""" -module foo -foo.bar - / - / -""") + out = self.parse_function_should_fail(""" + module foo + foo.bar + / + / + """) + msg = ( + "Function bar has an unsupported group configuration. " + "(Unexpected state 0.d)" + ) + self.assertIn(msg, out) + + def test_double_slash(self): + out = self.parse_function_should_fail(""" + module foo + foo.bar + a: int + / + b: int + / + """) + msg = "Function bar uses '/' more than once." + self.assertIn(msg, out) def test_mix_star_and_slash(self): - self.parse_function_should_fail(""" -module foo -foo.bar - x: int - y: int - * - z: int - / -""") + out = self.parse_function_should_fail(""" + module foo + foo.bar + x: int + y: int + * + z: int + / + """) + msg = ( + "Function bar mixes keyword-only and positional-only parameters, " + "which is unsupported." + ) + self.assertIn(msg, out) def test_parameters_not_permitted_after_slash_for_now(self): - self.parse_function_should_fail(""" -module foo -foo.bar - / - x: int -""") + out = self.parse_function_should_fail(""" + module foo + foo.bar + / + x: int + """) + msg = ( + "Function bar has an unsupported group configuration. " + "(Unexpected state 0.d)" + ) + self.assertIn(msg, out) def test_parameters_no_more_than_one_vararg(self): - s = self.parse_function_should_fail(""" -module foo -foo.bar - *vararg1: object - *vararg2: object -""") - self.assertEqual(s, "Error on line 0:\nToo many var args\n") + expected_msg = ( + "Error on line 0:\n" + "Too many var args\n" + ) + out = self.parse_function_should_fail(""" + module foo + foo.bar + *vararg1: object + *vararg2: object + """) + self.assertEqual(out, expected_msg) def test_function_not_at_column_0(self): function = self.parse_function(""" - module foo - foo.bar - x: int - Nested docstring here, goeth. - * - y: str - Not at column 0! -""") - self.assertEqual(""" -bar($module, /, x, *, y) --- + module foo + foo.bar + x: int + Nested docstring here, goeth. + * + y: str + Not at column 0! + """) + self.checkDocstring(function, """ + bar($module, /, x, *, y) + -- -Not at column 0! + Not at column 0! - x - Nested docstring here, goeth. -""".strip(), function.docstring) + x + Nested docstring here, goeth. + """) def test_directive(self): c = FakeClinic() @@ -772,46 +998,149 @@ def test_directive(self): def test_legacy_converters(self): block = self.parse('module os\nos.access\n path: "s"') module, function = block.signatures - self.assertIsInstance((function.parameters['path']).converter, clinic.str_converter) + conv = (function.parameters['path']).converter + self.assertIsInstance(conv, clinic.str_converter) def test_legacy_converters_non_string_constant_annotation(self): - expected_failure_message = """\ -Error on line 0: -Annotations must be either a name, a function call, or a string. -""" - - s = self.parse_function_should_fail('module os\nos.access\n path: 42') - self.assertEqual(s, expected_failure_message) - - s = self.parse_function_should_fail('module os\nos.access\n path: 42.42') - self.assertEqual(s, expected_failure_message) - - s = self.parse_function_should_fail('module os\nos.access\n path: 42j') - self.assertEqual(s, expected_failure_message) - - s = self.parse_function_should_fail('module os\nos.access\n path: b"42"') - self.assertEqual(s, expected_failure_message) + expected_failure_message = ( + "Error on line 0:\n" + "Annotations must be either a name, a function call, or a string.\n" + ) + dataset = ( + 'module os\nos.access\n path: 42', + 'module os\nos.access\n path: 42.42', + 'module os\nos.access\n path: 42j', + 'module os\nos.access\n path: b"42"', + ) + for block in dataset: + with self.subTest(block=block): + out = self.parse_function_should_fail(block) + self.assertEqual(out, expected_failure_message) def test_other_bizarre_things_in_annotations_fail(self): - expected_failure_message = """\ -Error on line 0: -Annotations must be either a name, a function call, or a string. -""" - - s = self.parse_function_should_fail( - 'module os\nos.access\n path: {"some": "dictionary"}' + expected_failure_message = ( + "Error on line 0:\n" + "Annotations must be either a name, a function call, or a string.\n" ) - self.assertEqual(s, expected_failure_message) + dataset = ( + 'module os\nos.access\n path: {"some": "dictionary"}', + 'module os\nos.access\n path: ["list", "of", "strings"]', + 'module os\nos.access\n path: (x for x in range(42))', + ) + for block in dataset: + with self.subTest(block=block): + out = self.parse_function_should_fail(block) + self.assertEqual(out, expected_failure_message) + + def test_kwarg_splats_disallowed_in_function_call_annotations(self): + expected_error_msg = ( + "Error on line 0:\n" + "Cannot use a kwarg splat in a function-call annotation\n" + ) + dataset = ( + 'module fo\nfo.barbaz\n o: bool(**{None: "bang!"})', + 'module fo\nfo.barbaz -> bool(**{None: "bang!"})', + 'module fo\nfo.barbaz -> bool(**{"bang": 42})', + 'module fo\nfo.barbaz\n o: bool(**{"bang": None})', + ) + for fn in dataset: + with self.subTest(fn=fn): + out = self.parse_function_should_fail(fn) + self.assertEqual(out, expected_error_msg) + + def test_self_param_placement(self): + expected_error_msg = ( + "Error on line 0:\n" + "A 'self' parameter, if specified, must be the very first thing " + "in the parameter block.\n" + ) + block = """ + module foo + foo.func + a: int + self: self(type="PyObject *") + """ + out = self.parse_function_should_fail(block) + self.assertEqual(out, expected_error_msg) + + def test_self_param_cannot_be_optional(self): + expected_error_msg = ( + "Error on line 0:\n" + "A 'self' parameter cannot be marked optional.\n" + ) + block = """ + module foo + foo.func + self: self(type="PyObject *") = None + """ + out = self.parse_function_should_fail(block) + self.assertEqual(out, expected_error_msg) + + def test_defining_class_param_placement(self): + expected_error_msg = ( + "Error on line 0:\n" + "A 'defining_class' parameter, if specified, must either be the " + "first thing in the parameter block, or come just after 'self'.\n" + ) + block = """ + module foo + foo.func + self: self(type="PyObject *") + a: int + cls: defining_class + """ + out = self.parse_function_should_fail(block) + self.assertEqual(out, expected_error_msg) + + def test_defining_class_param_cannot_be_optional(self): + expected_error_msg = ( + "Error on line 0:\n" + "A 'defining_class' parameter cannot be marked optional.\n" + ) + block = """ + module foo + foo.func + cls: defining_class(type="PyObject *") = None + """ + out = self.parse_function_should_fail(block) + self.assertEqual(out, expected_error_msg) - s = self.parse_function_should_fail( - 'module os\nos.access\n path: ["list", "of", "strings"]' + def test_slot_methods_cannot_access_defining_class(self): + block = """ + module foo + class Foo "" "" + Foo.__init__ + cls: defining_class + a: object + """ + msg = "Slot methods cannot access their defining class." + with self.assertRaisesRegex(ValueError, msg): + self.parse_function(block) + + def test_new_must_be_a_class_method(self): + expected_error_msg = ( + "Error on line 0:\n" + "__new__ must be a class method!\n" ) - self.assertEqual(s, expected_failure_message) + out = self.parse_function_should_fail(""" + module foo + class Foo "" "" + Foo.__new__ + """) + self.assertEqual(out, expected_error_msg) - s = self.parse_function_should_fail( - 'module os\nos.access\n path: (x for x in range(42))' + def test_init_must_be_a_normal_method(self): + expected_error_msg = ( + "Error on line 0:\n" + "__init__ must be a normal method, not a class or static method!\n" ) - self.assertEqual(s, expected_failure_message) + out = self.parse_function_should_fail(""" + module foo + class Foo "" "" + @classmethod + Foo.__init__ + """) + self.assertEqual(out, expected_error_msg) def test_unused_param(self): block = self.parse(""" @@ -872,25 +1201,32 @@ def test_scaffolding(self): self.assertEqual(repr(clinic.NULL), '') # test that fail fails + expected = ( + 'Error in file "clown.txt" on line 69:\n' + 'The igloos are melting!\n' + ) with support.captured_stdout() as stdout: with self.assertRaises(SystemExit): - clinic.fail('The igloos are melting!', filename='clown.txt', line_number=69) - self.assertEqual(stdout.getvalue(), 'Error in file "clown.txt" on line 69:\nThe igloos are melting!\n') + clinic.fail('The igloos are melting!', + filename='clown.txt', line_number=69) + actual = stdout.getvalue() + self.assertEqual(actual, expected) class ClinicExternalTest(TestCase): maxDiff = None def test_external(self): + CLINIC_TEST = 'clinic.test.c' # bpo-42398: Test that the destination file is left unchanged if the # content does not change. Moreover, check also that the file # modification time does not change in this case. - source = support.findfile('clinic.test') + source = support.findfile(CLINIC_TEST) with open(source, 'r', encoding='utf-8') as f: orig_contents = f.read() with os_helper.temp_dir() as tmp_dir: - testfile = os.path.join(tmp_dir, 'clinic.test.c') + testfile = os.path.join(tmp_dir, CLINIC_TEST) with open(testfile, 'w', encoding='utf-8') as f: f.write(orig_contents) old_mtime_ns = os.stat(testfile).st_mtime_ns @@ -1379,5 +1715,275 @@ def test_cloned_func_with_converter_exception_message(self): self.assertEqual(func(), name) +class PermutationTests(unittest.TestCase): + """Test permutation support functions.""" + + def test_permute_left_option_groups(self): + expected = ( + (), + (3,), + (2, 3), + (1, 2, 3), + ) + data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. + actual = tuple(clinic.permute_left_option_groups(data)) + self.assertEqual(actual, expected) + + def test_permute_right_option_groups(self): + expected = ( + (), + (1,), + (1, 2), + (1, 2, 3), + ) + data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. + actual = tuple(clinic.permute_right_option_groups(data)) + self.assertEqual(actual, expected) + + def test_permute_optional_groups(self): + empty = { + "left": (), "required": (), "right": (), + "expected": ((),), + } + noleft1 = { + "left": (), "required": ("b",), "right": ("c",), + "expected": ( + ("b",), + ("b", "c"), + ), + } + noleft2 = { + "left": (), "required": ("b", "c",), "right": ("d",), + "expected": ( + ("b", "c"), + ("b", "c", "d"), + ), + } + noleft3 = { + "left": (), "required": ("b", "c",), "right": ("d", "e"), + "expected": ( + ("b", "c"), + ("b", "c", "d"), + ("b", "c", "d", "e"), + ), + } + noright1 = { + "left": ("a",), "required": ("b",), "right": (), + "expected": ( + ("b",), + ("a", "b"), + ), + } + noright2 = { + "left": ("a",), "required": ("b", "c"), "right": (), + "expected": ( + ("b", "c"), + ("a", "b", "c"), + ), + } + noright3 = { + "left": ("a", "b"), "required": ("c",), "right": (), + "expected": ( + ("c",), + ("b", "c"), + ("a", "b", "c"), + ), + } + leftandright1 = { + "left": ("a",), "required": ("b",), "right": ("c",), + "expected": ( + ("b",), + ("a", "b"), # Prefer left. + ("a", "b", "c"), + ), + } + leftandright2 = { + "left": ("a", "b"), "required": ("c", "d"), "right": ("e", "f"), + "expected": ( + ("c", "d"), + ("b", "c", "d"), # Prefer left. + ("a", "b", "c", "d"), # Prefer left. + ("a", "b", "c", "d", "e"), + ("a", "b", "c", "d", "e", "f"), + ), + } + dataset = ( + empty, + noleft1, noleft2, noleft3, + noright1, noright2, noright3, + leftandright1, leftandright2, + ) + for params in dataset: + with self.subTest(**params): + left, required, right, expected = params.values() + permutations = clinic.permute_optional_groups(left, required, right) + actual = tuple(permutations) + self.assertEqual(actual, expected) + + +class FormatHelperTests(unittest.TestCase): + + def test_strip_leading_and_trailing_blank_lines(self): + dataset = ( + # Input lines, expected output. + ("a\nb", "a\nb"), + ("a\nb\n", "a\nb"), + ("a\nb ", "a\nb"), + ("\na\nb\n\n", "a\nb"), + ("\n\na\nb\n\n", "a\nb"), + ("\n\na\n\nb\n\n", "a\n\nb"), + # Note, leading whitespace is preserved: + (" a\nb", " a\nb"), + (" a\nb ", " a\nb"), + (" \n \n a\nb \n \n ", " a\nb"), + ) + for lines, expected in dataset: + with self.subTest(lines=lines, expected=expected): + out = clinic.strip_leading_and_trailing_blank_lines(lines) + self.assertEqual(out, expected) + + def test_normalize_snippet(self): + snippet = """ + one + two + three + """ + + # Expected outputs: + zero_indent = ( + "one\n" + "two\n" + "three" + ) + four_indent = ( + " one\n" + " two\n" + " three" + ) + eight_indent = ( + " one\n" + " two\n" + " three" + ) + expected_outputs = {0: zero_indent, 4: four_indent, 8: eight_indent} + for indent, expected in expected_outputs.items(): + with self.subTest(indent=indent): + actual = clinic.normalize_snippet(snippet, indent=indent) + self.assertEqual(actual, expected) + + def test_accumulator(self): + acc = clinic.text_accumulator() + self.assertEqual(acc.output(), "") + acc.append("a") + self.assertEqual(acc.output(), "a") + self.assertEqual(acc.output(), "") + acc.append("b") + self.assertEqual(acc.output(), "b") + self.assertEqual(acc.output(), "") + acc.append("c") + acc.append("d") + self.assertEqual(acc.output(), "cd") + self.assertEqual(acc.output(), "") + + def test_quoted_for_c_string(self): + dataset = ( + # input, expected + (r"abc", r"abc"), + (r"\abc", r"\\abc"), + (r"\a\bc", r"\\a\\bc"), + (r"\a\\bc", r"\\a\\\\bc"), + (r'"abc"', r'\"abc\"'), + (r"'a'", r"\'a\'"), + ) + for line, expected in dataset: + with self.subTest(line=line, expected=expected): + out = clinic.quoted_for_c_string(line) + self.assertEqual(out, expected) + + def test_rstrip_lines(self): + lines = ( + "a \n" + "b\n" + " c\n" + " d \n" + ) + expected = ( + "a\n" + "b\n" + " c\n" + " d\n" + ) + out = clinic.rstrip_lines(lines) + self.assertEqual(out, expected) + + def test_format_escape(self): + line = "{}, {a}" + expected = "{{}}, {{a}}" + out = clinic.format_escape(line) + self.assertEqual(out, expected) + + def test_indent_all_lines(self): + # Blank lines are expected to be unchanged. + self.assertEqual(clinic.indent_all_lines("", prefix="bar"), "") + + lines = ( + "one\n" + "two" # The missing newline is deliberate. + ) + expected = ( + "barone\n" + "bartwo" + ) + out = clinic.indent_all_lines(lines, prefix="bar") + self.assertEqual(out, expected) + + # If last line is empty, expect it to be unchanged. + lines = ( + "\n" + "one\n" + "two\n" + "" + ) + expected = ( + "bar\n" + "barone\n" + "bartwo\n" + "" + ) + out = clinic.indent_all_lines(lines, prefix="bar") + self.assertEqual(out, expected) + + def test_suffix_all_lines(self): + # Blank lines are expected to be unchanged. + self.assertEqual(clinic.suffix_all_lines("", suffix="foo"), "") + + lines = ( + "one\n" + "two" # The missing newline is deliberate. + ) + expected = ( + "onefoo\n" + "twofoo" + ) + out = clinic.suffix_all_lines(lines, suffix="foo") + self.assertEqual(out, expected) + + # If last line is empty, expect it to be unchanged. + lines = ( + "\n" + "one\n" + "two\n" + "" + ) + expected = ( + "foo\n" + "onefoo\n" + "twofoo\n" + "" + ) + out = clinic.suffix_all_lines(lines, suffix="foo") + self.assertEqual(out, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 8bf299382e9ca4..1b588826010717 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -777,7 +777,7 @@ def test_script_as_dev_fd(self): with os_helper.temp_dir() as work_dir: script_name = _make_test_script(work_dir, 'script.py', script) with open(script_name, "r") as fp: - p = spawn_python(f"/dev/fd/{fp.fileno()}", close_fds=False, pass_fds=(0,1,2,fp.fileno())) + p = spawn_python(f"/dev/fd/{fp.fileno()}", close_fds=True, pass_fds=(0,1,2,fp.fileno())) out, err = p.communicate() self.assertEqual(out, b"12345678912345678912345\n") diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 376175f90f63eb..91d7eaf997ae20 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -2822,14 +2822,15 @@ def test_binary_to_text_denylists_text_transforms(self): def test_custom_zlib_error_is_noted(self): # Check zlib codec gives a good error for malformed input msg = "decoding with 'zlib_codec' codec failed" - with self.assertRaises(Exception) as failure: + with self.assertRaises(zlib.error) as failure: codecs.decode(b"hello", "zlib_codec") self.assertEqual(msg, failure.exception.__notes__[0]) def test_custom_hex_error_is_noted(self): # Check hex codec gives a good error for malformed input + import binascii msg = "decoding with 'hex_codec' codec failed" - with self.assertRaises(Exception) as failure: + with self.assertRaises(binascii.Error) as failure: codecs.decode(b"hello", "hex_codec") self.assertEqual(msg, failure.exception.__notes__[0]) diff --git a/Lib/test/test_colorsys.py b/Lib/test/test_colorsys.py index a24e3adcb4b842..74d76294b0b4d4 100644 --- a/Lib/test/test_colorsys.py +++ b/Lib/test/test_colorsys.py @@ -69,6 +69,16 @@ def test_hls_values(self): self.assertTripleEqual(hls, colorsys.rgb_to_hls(*rgb)) self.assertTripleEqual(rgb, colorsys.hls_to_rgb(*hls)) + def test_hls_nearwhite(self): # gh-106498 + values = ( + # rgb, hls: these do not work in reverse + ((0.9999999999999999, 1, 1), (0.5, 1.0, 1.0)), + ((1, 0.9999999999999999, 0.9999999999999999), (0.0, 1.0, 1.0)), + ) + for rgb, hls in values: + self.assertTripleEqual(hls, colorsys.rgb_to_hls(*rgb)) + self.assertTripleEqual((1.0, 1.0, 1.0), colorsys.hls_to_rgb(*hls)) + def test_yiq_roundtrip(self): for r in frange(0.0, 1.0, 0.2): for g in frange(0.0, 1.0, 0.2): diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index a20cb844a293c9..39dbe234e765e8 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -1172,6 +1172,11 @@ def _crash(delay=None): faulthandler._sigsegv() +def _crash_with_data(data): + """Induces a segfault with dummy data in input.""" + _crash() + + def _exit(): """Induces a sys exit with exitcode 1.""" sys.exit(1) @@ -1371,6 +1376,19 @@ def test_shutdown_deadlock_pickle(self): # dangling threads executor_manager.join() + def test_crash_big_data(self): + # Test that there is a clean exception instad of a deadlock when a + # child process crashes while some data is being written into the + # queue. + # https://github.com/python/cpython/issues/94777 + self.executor.shutdown(wait=True) + data = "a" * support.PIPE_MAX_SIZE + with self.executor_type(max_workers=2, + mp_context=self.get_context()) as executor: + self.executor = executor # Allow clean up in fail_on_deadlock + with self.assertRaises(BrokenProcessPool): + list(executor.map(_crash_with_data, [data] * 10)) + create_executor_tests(ExecutorDeadlockTest, executor_mixins=(ProcessPoolForkMixin, diff --git a/Lib/test/test_cppext.py b/Lib/test/test_cppext.py index 4fb62d87e860fc..e2fedc9735079f 100644 --- a/Lib/test/test_cppext.py +++ b/Lib/test/test_cppext.py @@ -35,39 +35,20 @@ def test_build_cpp03(self): # the test uses venv+pip: skip if it's not available @support.requires_venv_with_pip() def check_build(self, std_cpp03, extension_name): - # Build in a temporary directory - with os_helper.temp_cwd(): - self._check_build(std_cpp03, extension_name) + venv_dir = 'env' + with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: + self._check_build(std_cpp03, extension_name, python_exe) - def _check_build(self, std_cpp03, extension_name): + def _check_build(self, std_cpp03, extension_name, python_exe): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py")) - venv_dir = 'env' - verbose = support.verbose - - # Create virtual environment to get setuptools - cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) - - # Get the Python executable of the venv - python_exe = 'python' - if sys.executable.endswith('.exe'): - python_exe += '.exe' - if MS_WINDOWS: - python = os.path.join(venv_dir, 'Scripts', python_exe) - else: - python = os.path.join(venv_dir, 'bin', python_exe) - def run_cmd(operation, cmd): env = os.environ.copy() env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11' env['CPYTHON_TEST_EXT_NAME'] = extension_name - if verbose: + if support.verbose: print('Run:', ' '.join(cmd)) subprocess.run(cmd, check=True, env=env) else: @@ -81,14 +62,8 @@ def run_cmd(operation, cmd): self.fail( f"{operation} failed with exit code {proc.returncode}") - cmd = [python, '-X', 'dev', - '-m', 'pip', 'install', - support.findfile('setuptools-67.6.1-py3-none-any.whl'), - support.findfile('wheel-0.40.0-py3-none-any.whl')] - run_cmd('Install build dependencies', cmd) - # Build and install the C++ extension - cmd = [python, '-X', 'dev', + cmd = [python_exe, '-X', 'dev', '-m', 'pip', 'install', '--no-build-isolation', os.path.abspath(pkg_dir)] run_cmd('Install', cmd) @@ -96,14 +71,14 @@ def run_cmd(operation, cmd): # Do a reference run. Until we test that running python # doesn't leak references (gh-94755), run it so one can manually check # -X showrefcount results against this baseline. - cmd = [python, + cmd = [python_exe, '-X', 'dev', '-X', 'showrefcount', '-c', 'pass'] run_cmd('Reference run', cmd) # Import the C++ extension - cmd = [python, + cmd = [python_exe, '-X', 'dev', '-X', 'showrefcount', '-c', f"import {extension_name}"] diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 484b8f8e3a365c..3056fe84dac5dd 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -66,6 +66,26 @@ def test_second_profiler(self): self.assertRaises(ValueError, pr2.enable) pr.disable() + def test_throw(self): + """ + gh-106152 + generator.throw() should trigger a call in cProfile + In the any() call below, there should be two entries for the generator: + * one for the call to __next__ which gets a True and terminates any + * one when the generator is garbage collected which will effectively + do a throw. + """ + pr = self.profilerclass() + pr.enable() + any(a == 1 for a in (1, 2)) + pr.disable() + pr.create_stats() + + for func, (cc, nc, _, _, _) in pr.stats.items(): + if func[2] == "": + self.assertEqual(cc, 2) + self.assertEqual(nc, 2) + class TestCommandLine(unittest.TestCase): def test_sort(self): diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 3ab837e4f95681..31bc108e7712ea 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -1364,26 +1364,33 @@ def test_move_left(self): self.mock_win.reset_mock() self.textbox.do_command(curses.KEY_LEFT) self.mock_win.move.assert_called_with(1, 0) + self.mock_win.reset_mock() + + def test_move_right(self): + """Test moving the cursor right.""" + self.mock_win.reset_mock() self.textbox.do_command(curses.KEY_RIGHT) self.mock_win.move.assert_called_with(1, 2) self.mock_win.reset_mock() - def test_move_left(self): - """Test moving the cursor left.""" + def test_move_left_and_right(self): + """Test moving the cursor left and then right.""" self.mock_win.reset_mock() + self.textbox.do_command(curses.KEY_LEFT) + self.mock_win.move.assert_called_with(1, 0) self.textbox.do_command(curses.KEY_RIGHT) self.mock_win.move.assert_called_with(1, 2) self.mock_win.reset_mock() def test_move_up(self): - """Test moving the cursor left.""" + """Test moving the cursor up.""" self.mock_win.reset_mock() self.textbox.do_command(curses.KEY_UP) self.mock_win.move.assert_called_with(0, 1) self.mock_win.reset_mock() def test_move_down(self): - """Test moving the cursor left.""" + """Test moving the cursor down.""" self.mock_win.reset_mock() self.textbox.do_command(curses.KEY_DOWN) self.mock_win.move.assert_called_with(2, 1) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 67ccaab40c5edc..749496e3e9455e 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -20,7 +20,7 @@ This test module can be called from command line with one parameter (Arithmetic or Behaviour) to test each part, or without parameter to test both parts. If -you're working through IDLE, you can import this test module and call test_main() +you're working through IDLE, you can import this test module and call test() with the corresponding argument. """ @@ -32,7 +32,7 @@ import unittest import numbers import locale -from test.support import (run_unittest, run_doctest, is_resource_enabled, +from test.support import (is_resource_enabled, requires_IEEE_754, requires_docstrings, requires_legacy_unicode_capi, check_sanitizer) from test.support import (TestFailed, @@ -62,6 +62,7 @@ fractions = {C:cfractions, P:pfractions} sys.modules['decimal'] = orig_sys_decimal +requires_cdecimal = unittest.skipUnless(C, "test requires C version") # Useful Test Constant Signals = { @@ -99,7 +100,7 @@ def assert_signals(cls, context, attr, expected): ] # Tests are built around these assumed context defaults. -# test_main() restores the original context. +# test() restores the original context. ORIGINAL_CONTEXT = { C: C.getcontext().copy() if C else None, P: P.getcontext().copy() @@ -133,7 +134,7 @@ def init(m): EXTRA_FUNCTIONALITY, "test requires regular build") -class IBMTestCases(unittest.TestCase): +class IBMTestCases: """Class which tests the Decimal class against the IBM test cases.""" def setUp(self): @@ -488,14 +489,10 @@ def change_max_exponent(self, exp): def change_clamp(self, clamp): self.context.clamp = clamp -class CIBMTestCases(IBMTestCases): - decimal = C -class PyIBMTestCases(IBMTestCases): - decimal = P # The following classes test the behaviour of Decimal according to PEP 327 -class ExplicitConstructionTest(unittest.TestCase): +class ExplicitConstructionTest: '''Unit tests for Explicit Construction cases of Decimal.''' def test_explicit_empty(self): @@ -838,12 +835,13 @@ def test_unicode_digits(self): for input, expected in test_values.items(): self.assertEqual(str(Decimal(input)), expected) -class CExplicitConstructionTest(ExplicitConstructionTest): +@requires_cdecimal +class CExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): decimal = C -class PyExplicitConstructionTest(ExplicitConstructionTest): +class PyExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): decimal = P -class ImplicitConstructionTest(unittest.TestCase): +class ImplicitConstructionTest: '''Unit tests for Implicit Construction cases of Decimal.''' def test_implicit_from_None(self): @@ -920,12 +918,13 @@ def __ne__(self, other): self.assertEqual(eval('Decimal(10)' + sym + 'E()'), '10' + rop + 'str') -class CImplicitConstructionTest(ImplicitConstructionTest): +@requires_cdecimal +class CImplicitConstructionTest(ImplicitConstructionTest, unittest.TestCase): decimal = C -class PyImplicitConstructionTest(ImplicitConstructionTest): +class PyImplicitConstructionTest(ImplicitConstructionTest, unittest.TestCase): decimal = P -class FormatTest(unittest.TestCase): +class FormatTest: '''Unit tests for the format function.''' def test_formatting(self): Decimal = self.decimal.Decimal @@ -1262,12 +1261,13 @@ def __init__(self, a): a = A.from_float(42) self.assertEqual(self.decimal.Decimal, a.a_type) -class CFormatTest(FormatTest): +@requires_cdecimal +class CFormatTest(FormatTest, unittest.TestCase): decimal = C -class PyFormatTest(FormatTest): +class PyFormatTest(FormatTest, unittest.TestCase): decimal = P -class ArithmeticOperatorsTest(unittest.TestCase): +class ArithmeticOperatorsTest: '''Unit tests for all arithmetic operators, binary and unary.''' def test_addition(self): @@ -1523,14 +1523,17 @@ def test_nan_comparisons(self): equality_ops = operator.eq, operator.ne # results when InvalidOperation is not trapped - for x, y in qnan_pairs + snan_pairs: - for op in order_ops + equality_ops: - got = op(x, y) - expected = True if op is operator.ne else False - self.assertIs(expected, got, - "expected {0!r} for operator.{1}({2!r}, {3!r}); " - "got {4!r}".format( - expected, op.__name__, x, y, got)) + with localcontext() as ctx: + ctx.traps[InvalidOperation] = 0 + + for x, y in qnan_pairs + snan_pairs: + for op in order_ops + equality_ops: + got = op(x, y) + expected = True if op is operator.ne else False + self.assertIs(expected, got, + "expected {0!r} for operator.{1}({2!r}, {3!r}); " + "got {4!r}".format( + expected, op.__name__, x, y, got)) # repeat the above, but this time trap the InvalidOperation with localcontext() as ctx: @@ -1562,9 +1565,10 @@ def test_copy_sign(self): self.assertEqual(Decimal(1).copy_sign(-2), d) self.assertRaises(TypeError, Decimal(1).copy_sign, '-2') -class CArithmeticOperatorsTest(ArithmeticOperatorsTest): +@requires_cdecimal +class CArithmeticOperatorsTest(ArithmeticOperatorsTest, unittest.TestCase): decimal = C -class PyArithmeticOperatorsTest(ArithmeticOperatorsTest): +class PyArithmeticOperatorsTest(ArithmeticOperatorsTest, unittest.TestCase): decimal = P # The following are two functions used to test threading in the next class @@ -1654,7 +1658,7 @@ def thfunc2(cls): @threading_helper.requires_working_threading() -class ThreadingTest(unittest.TestCase): +class ThreadingTest: '''Unit tests for thread local contexts in Decimal.''' # Take care executing this test from IDLE, there's an issue in threading @@ -1699,13 +1703,14 @@ def test_threading(self): DefaultContext.Emin = save_emin -class CThreadingTest(ThreadingTest): +@requires_cdecimal +class CThreadingTest(ThreadingTest, unittest.TestCase): decimal = C -class PyThreadingTest(ThreadingTest): +class PyThreadingTest(ThreadingTest, unittest.TestCase): decimal = P -class UsabilityTest(unittest.TestCase): +class UsabilityTest: '''Unit tests for Usability cases of Decimal.''' def test_comparison_operators(self): @@ -2521,9 +2526,10 @@ def test_conversions_from_int(self): self.assertEqual(Decimal(-12).fma(45, Decimal(67)), Decimal(-12).fma(Decimal(45), Decimal(67))) -class CUsabilityTest(UsabilityTest): +@requires_cdecimal +class CUsabilityTest(UsabilityTest, unittest.TestCase): decimal = C -class PyUsabilityTest(UsabilityTest): +class PyUsabilityTest(UsabilityTest, unittest.TestCase): decimal = P def setUp(self): @@ -2535,7 +2541,7 @@ def tearDown(self): sys.set_int_max_str_digits(self._previous_int_limit) super().tearDown() -class PythonAPItests(unittest.TestCase): +class PythonAPItests: def test_abc(self): Decimal = self.decimal.Decimal @@ -2884,12 +2890,13 @@ def test_exception_hierarchy(self): self.assertTrue(issubclass(decimal.DivisionUndefined, ZeroDivisionError)) self.assertTrue(issubclass(decimal.InvalidContext, InvalidOperation)) -class CPythonAPItests(PythonAPItests): +@requires_cdecimal +class CPythonAPItests(PythonAPItests, unittest.TestCase): decimal = C -class PyPythonAPItests(PythonAPItests): +class PyPythonAPItests(PythonAPItests, unittest.TestCase): decimal = P -class ContextAPItests(unittest.TestCase): +class ContextAPItests: def test_none_args(self): Context = self.decimal.Context @@ -3635,12 +3642,13 @@ def test_to_integral_value(self): self.assertRaises(TypeError, c.to_integral_value, '10') self.assertRaises(TypeError, c.to_integral_value, 10, 'x') -class CContextAPItests(ContextAPItests): +@requires_cdecimal +class CContextAPItests(ContextAPItests, unittest.TestCase): decimal = C -class PyContextAPItests(ContextAPItests): +class PyContextAPItests(ContextAPItests, unittest.TestCase): decimal = P -class ContextWithStatement(unittest.TestCase): +class ContextWithStatement: # Can't do these as docstrings until Python 2.6 # as doctest can't handle __future__ statements @@ -3704,9 +3712,13 @@ def test_localcontext_kwargs(self): def test_local_context_kwargs_does_not_overwrite_existing_argument(self): ctx = self.decimal.getcontext() - ctx.prec = 28 + orig_prec = ctx.prec with self.decimal.localcontext(prec=10) as ctx2: - self.assertEqual(ctx.prec, 28) + self.assertEqual(ctx2.prec, 10) + self.assertEqual(ctx.prec, orig_prec) + with self.decimal.localcontext(prec=20) as ctx2: + self.assertEqual(ctx2.prec, 20) + self.assertEqual(ctx.prec, orig_prec) def test_nested_with_statements(self): # Use a copy of the supplied context in the block @@ -3800,12 +3812,13 @@ def test_with_statements_gc3(self): self.assertEqual(c4.prec, 4) del c4 -class CContextWithStatement(ContextWithStatement): +@requires_cdecimal +class CContextWithStatement(ContextWithStatement, unittest.TestCase): decimal = C -class PyContextWithStatement(ContextWithStatement): +class PyContextWithStatement(ContextWithStatement, unittest.TestCase): decimal = P -class ContextFlags(unittest.TestCase): +class ContextFlags: def test_flags_irrelevant(self): # check that the result (numeric result + flags raised) of an @@ -4072,12 +4085,13 @@ def test_float_operation_default(self): self.assertTrue(context.traps[FloatOperation]) self.assertTrue(context.traps[Inexact]) -class CContextFlags(ContextFlags): +@requires_cdecimal +class CContextFlags(ContextFlags, unittest.TestCase): decimal = C -class PyContextFlags(ContextFlags): +class PyContextFlags(ContextFlags, unittest.TestCase): decimal = P -class SpecialContexts(unittest.TestCase): +class SpecialContexts: """Test the context templates.""" def test_context_templates(self): @@ -4157,12 +4171,13 @@ def test_default_context(self): if ex: raise ex -class CSpecialContexts(SpecialContexts): +@requires_cdecimal +class CSpecialContexts(SpecialContexts, unittest.TestCase): decimal = C -class PySpecialContexts(SpecialContexts): +class PySpecialContexts(SpecialContexts, unittest.TestCase): decimal = P -class ContextInputValidation(unittest.TestCase): +class ContextInputValidation: def test_invalid_context(self): Context = self.decimal.Context @@ -4224,12 +4239,13 @@ def test_invalid_context(self): self.assertRaises(TypeError, Context, flags=(0,1)) self.assertRaises(TypeError, Context, traps=(1,0)) -class CContextInputValidation(ContextInputValidation): +@requires_cdecimal +class CContextInputValidation(ContextInputValidation, unittest.TestCase): decimal = C -class PyContextInputValidation(ContextInputValidation): +class PyContextInputValidation(ContextInputValidation, unittest.TestCase): decimal = P -class ContextSubclassing(unittest.TestCase): +class ContextSubclassing: def test_context_subclassing(self): decimal = self.decimal @@ -4338,12 +4354,14 @@ def __init__(self, prec=None, rounding=None, Emin=None, Emax=None, for signal in OrderedSignals[decimal]: self.assertFalse(c.traps[signal]) -class CContextSubclassing(ContextSubclassing): +@requires_cdecimal +class CContextSubclassing(ContextSubclassing, unittest.TestCase): decimal = C -class PyContextSubclassing(ContextSubclassing): +class PyContextSubclassing(ContextSubclassing, unittest.TestCase): decimal = P @skip_if_extra_functionality +@requires_cdecimal class CheckAttributes(unittest.TestCase): def test_module_attributes(self): @@ -4373,7 +4391,7 @@ def test_decimal_attributes(self): y = [s for s in dir(C.Decimal(9)) if '__' in s or not s.startswith('_')] self.assertEqual(set(x) - set(y), set()) -class Coverage(unittest.TestCase): +class Coverage: def test_adjusted(self): Decimal = self.decimal.Decimal @@ -4630,9 +4648,10 @@ def test_copy(self): y = c.copy_sign(x, 1) self.assertEqual(y, -x) -class CCoverage(Coverage): +@requires_cdecimal +class CCoverage(Coverage, unittest.TestCase): decimal = C -class PyCoverage(Coverage): +class PyCoverage(Coverage, unittest.TestCase): decimal = P def setUp(self): @@ -4885,6 +4904,7 @@ def test_constants(self): self.assertEqual(C.DecTraps, C.DecErrors|C.DecOverflow|C.DecUnderflow) +@requires_cdecimal class CWhitebox(unittest.TestCase): """Whitebox testing for _decimal""" @@ -5663,7 +5683,7 @@ def test_maxcontext_exact_arith(self): @requires_docstrings -@unittest.skipUnless(C, "test requires C version") +@requires_cdecimal class SignatureTest(unittest.TestCase): """Function signatures""" @@ -5799,52 +5819,10 @@ def doit(ty): doit('Context') -all_tests = [ - CExplicitConstructionTest, PyExplicitConstructionTest, - CImplicitConstructionTest, PyImplicitConstructionTest, - CFormatTest, PyFormatTest, - CArithmeticOperatorsTest, PyArithmeticOperatorsTest, - CThreadingTest, PyThreadingTest, - CUsabilityTest, PyUsabilityTest, - CPythonAPItests, PyPythonAPItests, - CContextAPItests, PyContextAPItests, - CContextWithStatement, PyContextWithStatement, - CContextFlags, PyContextFlags, - CSpecialContexts, PySpecialContexts, - CContextInputValidation, PyContextInputValidation, - CContextSubclassing, PyContextSubclassing, - CCoverage, PyCoverage, - CFunctionality, PyFunctionality, - CWhitebox, PyWhitebox, - CIBMTestCases, PyIBMTestCases, -] - -# Delete C tests if _decimal.so is not present. -if not C: - all_tests = all_tests[1::2] -else: - all_tests.insert(0, CheckAttributes) - all_tests.insert(1, SignatureTest) - - -def test_main(arith=None, verbose=None, todo_tests=None, debug=None): - """ Execute the tests. - - Runs all arithmetic tests if arith is True or if the "decimal" resource - is enabled in regrtest.py - """ - - init(C) - init(P) - global TEST_ALL, DEBUG - TEST_ALL = arith if arith is not None else is_resource_enabled('decimal') - DEBUG = debug - - if todo_tests is None: - test_classes = all_tests - else: - test_classes = [CIBMTestCases, PyIBMTestCases] - +def load_tests(loader, tests, pattern): + if TODO_TESTS is not None: + # Run only Arithmetic tests + tests = loader.suiteClass() # Dynamically build custom test definition for each file in the test # directory and add the definitions to the DecimalTest class. This # procedure insures that new files do not get skipped. @@ -5852,34 +5830,69 @@ def test_main(arith=None, verbose=None, todo_tests=None, debug=None): if '.decTest' not in filename or filename.startswith("."): continue head, tail = filename.split('.') - if todo_tests is not None and head not in todo_tests: + if TODO_TESTS is not None and head not in TODO_TESTS: continue tester = lambda self, f=filename: self.eval_file(directory + f) - setattr(CIBMTestCases, 'test_' + head, tester) - setattr(PyIBMTestCases, 'test_' + head, tester) + setattr(IBMTestCases, 'test_' + head, tester) del filename, head, tail, tester + for prefix, mod in ('C', C), ('Py', P): + if not mod: + continue + test_class = type(prefix + 'IBMTestCases', + (IBMTestCases, unittest.TestCase), + {'decimal': mod}) + tests.addTest(loader.loadTestsFromTestCase(test_class)) + + if TODO_TESTS is None: + from doctest import DocTestSuite, IGNORE_EXCEPTION_DETAIL + for mod in C, P: + if not mod: + continue + def setUp(slf, mod=mod): + sys.modules['decimal'] = mod + def tearDown(slf): + sys.modules['decimal'] = orig_sys_decimal + optionflags = IGNORE_EXCEPTION_DETAIL if mod is C else 0 + sys.modules['decimal'] = mod + tests.addTest(DocTestSuite(mod, setUp=setUp, tearDown=tearDown, + optionflags=optionflags)) + sys.modules['decimal'] = orig_sys_decimal + return tests + +def setUpModule(): + init(C) + init(P) + global TEST_ALL + TEST_ALL = ARITH if ARITH is not None else is_resource_enabled('decimal') + +def tearDownModule(): + if C: C.setcontext(ORIGINAL_CONTEXT[C]) + P.setcontext(ORIGINAL_CONTEXT[P]) + if not C: + warnings.warn('C tests skipped: no module named _decimal.', + UserWarning) + if not orig_sys_decimal is sys.modules['decimal']: + raise TestFailed("Internal error: unbalanced number of changes to " + "sys.modules['decimal'].") + + +ARITH = None +TEST_ALL = True +TODO_TESTS = None +DEBUG = False + +def test(arith=None, verbose=None, todo_tests=None, debug=None): + """ Execute the tests. + Runs all arithmetic tests if arith is True or if the "decimal" resource + is enabled in regrtest.py + """ - try: - run_unittest(*test_classes) - if todo_tests is None: - from doctest import IGNORE_EXCEPTION_DETAIL - savedecimal = sys.modules['decimal'] - if C: - sys.modules['decimal'] = C - run_doctest(C, verbose, optionflags=IGNORE_EXCEPTION_DETAIL) - sys.modules['decimal'] = P - run_doctest(P, verbose) - sys.modules['decimal'] = savedecimal - finally: - if C: C.setcontext(ORIGINAL_CONTEXT[C]) - P.setcontext(ORIGINAL_CONTEXT[P]) - if not C: - warnings.warn('C tests skipped: no module named _decimal.', - UserWarning) - if not orig_sys_decimal is sys.modules['decimal']: - raise TestFailed("Internal error: unbalanced number of changes to " - "sys.modules['decimal'].") + global ARITH, TODO_TESTS, DEBUG + ARITH = arith + TODO_TESTS = todo_tests + DEBUG = debug + unittest.main(__name__, verbosity=2 if verbose else 1, exit=False, argv=[__name__]) if __name__ == '__main__': @@ -5890,8 +5903,8 @@ def test_main(arith=None, verbose=None, todo_tests=None, debug=None): (opt, args) = p.parse_args() if opt.skip: - test_main(arith=False, verbose=True) + test(arith=False, verbose=True) elif args: - test_main(arith=True, verbose=True, todo_tests=args, debug=opt.debug) + test(arith=True, verbose=True, todo_tests=args, debug=opt.debug) else: - test_main(arith=True, verbose=True) + test(arith=True, verbose=True) diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 44b405740c4403..5238944d6b4788 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -3319,15 +3319,90 @@ def test_getaddresses(self): [('Al Person', 'aperson@dom.ain'), ('Bud Person', 'bperson@dom.ain')]) + def test_getaddresses_parsing_errors(self): + """Test for parsing errors from CVE-2023-27043""" + eq = self.assertEqual + eq(utils.getaddresses(['alice@example.org(']), + [('', '')]) + eq(utils.getaddresses(['alice@example.org)']), + [('', '')]) + eq(utils.getaddresses(['alice@example.org<']), + [('', '')]) + eq(utils.getaddresses(['alice@example.org>']), + [('', '')]) + eq(utils.getaddresses(['alice@example.org@']), + [('', '')]) + eq(utils.getaddresses(['alice@example.org,']), + [('', 'alice@example.org'), ('', 'bob@example.com')]) + eq(utils.getaddresses(['alice@example.org;']), + [('', '')]) + eq(utils.getaddresses(['alice@example.org:']), + [('', '')]) + eq(utils.getaddresses(['alice@example.org.']), + [('', '')]) + eq(utils.getaddresses(['alice@example.org"']), + [('', '')]) + eq(utils.getaddresses(['alice@example.org[']), + [('', '')]) + eq(utils.getaddresses(['alice@example.org]']), + [('', '')]) + + def test_parseaddr_parsing_errors(self): + """Test for parsing errors from CVE-2023-27043""" + eq = self.assertEqual + eq(utils.parseaddr(['alice@example.org(']), + ('', '')) + eq(utils.parseaddr(['alice@example.org)']), + ('', '')) + eq(utils.parseaddr(['alice@example.org<']), + ('', '')) + eq(utils.parseaddr(['alice@example.org>']), + ('', '')) + eq(utils.parseaddr(['alice@example.org@']), + ('', '')) + eq(utils.parseaddr(['alice@example.org,']), + ('', '')) + eq(utils.parseaddr(['alice@example.org;']), + ('', '')) + eq(utils.parseaddr(['alice@example.org:']), + ('', '')) + eq(utils.parseaddr(['alice@example.org.']), + ('', '')) + eq(utils.parseaddr(['alice@example.org"']), + ('', '')) + eq(utils.parseaddr(['alice@example.org[']), + ('', '')) + eq(utils.parseaddr(['alice@example.org]']), + ('', '')) + def test_getaddresses_nasty(self): eq = self.assertEqual eq(utils.getaddresses(['foo: ;']), [('', '')]) - eq(utils.getaddresses( - ['[]*-- =~$']), - [('', ''), ('', ''), ('', '*--')]) + eq(utils.getaddresses(['[]*-- =~$']), [('', '')]) eq(utils.getaddresses( ['foo: ;', '"Jason R. Mastaler" ']), [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]) + eq(utils.getaddresses( + [r'Pete(A nice \) chap) ']), + [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]) + eq(utils.getaddresses( + ['(Empty list)(start)Undisclosed recipients :(nobody(I know))']), + [('', '')]) + eq(utils.getaddresses( + ['Mary <@machine.tld:mary@example.net>, , jdoe@test . example']), + [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]) + eq(utils.getaddresses( + ['John Doe ']), + [('John Doe (comment)', 'jdoe@machine.example')]) + eq(utils.getaddresses( + ['"Mary Smith: Personal Account" ']), + [('Mary Smith: Personal Account', 'smith@home.example')]) + eq(utils.getaddresses( + ['Undisclosed recipients:;']), + [('', '')]) + eq(utils.getaddresses( + [r', "Giant; \"Big\" Box" ']), + [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]) def test_getaddresses_embedded_comment(self): """Test proper handling of a nested comment""" diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 4c754bf40fc300..d3f396f02e7a72 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -696,14 +696,16 @@ def subtype_as_add(self, method, subtype, outcome): self.assertIsNone(part['Content-Disposition']) class _TestSetRaisingContentManager: + class CustomError(Exception): + pass def set_content(self, msg, content, *args, **kw): - raise Exception('test') + raise self.CustomError('test') def test_default_content_manager_for_add_comes_from_policy(self): cm = self._TestSetRaisingContentManager() m = self.message(policy=self.policy.clone(content_manager=cm)) for method in ('add_related', 'add_alternative', 'add_attachment'): - with self.assertRaises(Exception) as ar: + with self.assertRaises(self._TestSetRaisingContentManager.CustomError) as ar: getattr(m, method)('') self.assertEqual(str(ar.exception), 'test') diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index b4ac3abfc7006f..dbdc639d62ec1f 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -804,9 +804,17 @@ def test_copy(self): TE = self.MainEnum copied = copy.copy(TE) self.assertEqual(copied, TE) + self.assertIs(copied, TE) deep = copy.deepcopy(TE) self.assertEqual(deep, TE) + self.assertIs(deep, TE) + def test_copy_member(self): + TE = self.MainEnum + copied = copy.copy(TE.first) + self.assertIs(copied, TE.first) + deep = copy.deepcopy(TE.first) + self.assertIs(deep, TE.first) class _FlagTests: @@ -818,6 +826,89 @@ def test_default_missing_with_wrong_type_value(self): self.MainEnum('RED') self.assertIs(ctx.exception.__context__, None) + def test_closed_invert_expectations(self): + class ClosedAB(self.enum_type): + A = 1 + B = 2 + MASK = 3 + A, B = ClosedAB + AB_MASK = ClosedAB.MASK + # + self.assertIs(~A, B) + self.assertIs(~B, A) + self.assertIs(~(A|B), ClosedAB(0)) + self.assertIs(~AB_MASK, ClosedAB(0)) + self.assertIs(~ClosedAB(0), (A|B)) + # + class ClosedXYZ(self.enum_type): + X = 4 + Y = 2 + Z = 1 + MASK = 7 + X, Y, Z = ClosedXYZ + XYZ_MASK = ClosedXYZ.MASK + # + self.assertIs(~X, Y|Z) + self.assertIs(~Y, X|Z) + self.assertIs(~Z, X|Y) + self.assertIs(~(X|Y), Z) + self.assertIs(~(X|Z), Y) + self.assertIs(~(Y|Z), X) + self.assertIs(~(X|Y|Z), ClosedXYZ(0)) + self.assertIs(~XYZ_MASK, ClosedXYZ(0)) + self.assertIs(~ClosedXYZ(0), (X|Y|Z)) + + def test_open_invert_expectations(self): + class OpenAB(self.enum_type): + A = 1 + B = 2 + MASK = 255 + A, B = OpenAB + AB_MASK = OpenAB.MASK + # + if OpenAB._boundary_ in (EJECT, KEEP): + self.assertIs(~A, OpenAB(254)) + self.assertIs(~B, OpenAB(253)) + self.assertIs(~(A|B), OpenAB(252)) + self.assertIs(~AB_MASK, OpenAB(0)) + self.assertIs(~OpenAB(0), AB_MASK) + else: + self.assertIs(~A, B) + self.assertIs(~B, A) + self.assertIs(~(A|B), OpenAB(0)) + self.assertIs(~AB_MASK, OpenAB(0)) + self.assertIs(~OpenAB(0), (A|B)) + # + class OpenXYZ(self.enum_type): + X = 4 + Y = 2 + Z = 1 + MASK = 31 + X, Y, Z = OpenXYZ + XYZ_MASK = OpenXYZ.MASK + # + if OpenXYZ._boundary_ in (EJECT, KEEP): + self.assertIs(~X, OpenXYZ(27)) + self.assertIs(~Y, OpenXYZ(29)) + self.assertIs(~Z, OpenXYZ(30)) + self.assertIs(~(X|Y), OpenXYZ(25)) + self.assertIs(~(X|Z), OpenXYZ(26)) + self.assertIs(~(Y|Z), OpenXYZ(28)) + self.assertIs(~(X|Y|Z), OpenXYZ(24)) + self.assertIs(~XYZ_MASK, OpenXYZ(0)) + self.assertTrue(~OpenXYZ(0), XYZ_MASK) + else: + self.assertIs(~X, Y|Z) + self.assertIs(~Y, X|Z) + self.assertIs(~Z, X|Y) + self.assertIs(~(X|Y), Z) + self.assertIs(~(X|Z), Y) + self.assertIs(~(Y|Z), X) + self.assertIs(~(X|Y|Z), OpenXYZ(0)) + self.assertIs(~XYZ_MASK, OpenXYZ(0)) + self.assertTrue(~OpenXYZ(0), (X|Y|Z)) + + class TestPlainEnum(_EnumTests, _PlainOutputTests, unittest.TestCase): enum_type = Enum @@ -3088,22 +3179,6 @@ def test_xor(self): self.assertIs(Open.RO ^ Open.CE, Open.CE) self.assertIs(Open.CE ^ Open.CE, Open.RO) - def test_invert(self): - Perm = self.Perm - RW = Perm.R | Perm.W - RX = Perm.R | Perm.X - WX = Perm.W | Perm.X - RWX = Perm.R | Perm.W | Perm.X - values = list(Perm) + [RW, RX, WX, RWX, Perm(0)] - for i in values: - self.assertIs(type(~i), Perm) - self.assertEqual(~~i, i) - for i in Perm: - self.assertIs(~~i, i) - Open = self.Open - self.assertIs(Open.WO & ~Open.WO, Open.RO) - self.assertIs((Open.WO|Open.CE) & ~Open.WO, Open.CE) - def test_bool(self): Perm = self.Perm for f in Perm: diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 031b94d8d58a39..cb14bba2602def 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -13,9 +13,10 @@ import types import decimal import unittest +import warnings from test import support from test.support.os_helper import temp_cwd -from test.support.script_helper import assert_python_failure +from test.support.script_helper import assert_python_failure, assert_python_ok a_global = 'global variable' @@ -495,6 +496,24 @@ def test_ast_line_numbers_with_parentheses(self): self.assertEqual(wat2.end_col_offset, 17) self.assertEqual(fstring.end_col_offset, 18) + def test_ast_fstring_empty_format_spec(self): + expr = "f'{expr:}'" + + mod = ast.parse(expr) + self.assertEqual(type(mod), ast.Module) + self.assertEqual(len(mod.body), 1) + + fstring = mod.body[0].value + self.assertEqual(type(fstring), ast.JoinedStr) + self.assertEqual(len(fstring.values), 1) + + fv = fstring.values[0] + self.assertEqual(type(fv), ast.FormattedValue) + + format_spec = fv.format_spec + self.assertEqual(type(format_spec), ast.JoinedStr) + self.assertEqual(len(format_spec.values), 0) + def test_docstring(self): def f(): f'''Not a docstring''' @@ -772,7 +791,7 @@ def __format__(self, format_spec): self.assertEqual(f'{CustomFormat():\n}', '\n') self.assertEqual(f'{CustomFormat():\u2603}', '☃') with self.assertWarns(SyntaxWarning): - exec('f"{F():¯\_(ツ)_/¯}"', {'F': CustomFormat}) + exec(r'f"{F():¯\_(ツ)_/¯}"', {'F': CustomFormat}) def test_side_effect_order(self): class X: @@ -904,9 +923,12 @@ def test_backslashes_in_string_part(self): self.assertEqual(f'2\x203', '2 3') self.assertEqual(f'\x203', ' 3') - with self.assertWarns(DeprecationWarning): # invalid escape sequence + with self.assertWarns(SyntaxWarning): # invalid escape sequence value = eval(r"f'\{6*7}'") self.assertEqual(value, '\\42') + with self.assertWarns(SyntaxWarning): # invalid escape sequence + value = eval(r"f'\g'") + self.assertEqual(value, '\\g') self.assertEqual(f'\\{6*7}', '\\42') self.assertEqual(fr'\{6*7}', '\\42') @@ -1034,7 +1056,7 @@ def test_fstring_backslash_before_double_bracket(self): ] for case, expected_result in deprecated_cases: with self.subTest(case=case, expected_result=expected_result): - with self.assertWarns(DeprecationWarning): + with self.assertWarns(SyntaxWarning): result = eval(case) self.assertEqual(result, expected_result) self.assertEqual(fr'\{{\}}', '\\{\\}') @@ -1043,6 +1065,12 @@ def test_fstring_backslash_before_double_bracket(self): self.assertEqual(fr'\}}{1+1}', '\\}2') self.assertEqual(fr'{1+1}\}}', '2\\}') + def test_fstring_backslash_before_double_bracket_warns_once(self): + with self.assertWarns(SyntaxWarning) as w: + eval(r"f'\{{'") + self.assertEqual(len(w.warnings), 1) + self.assertEqual(w.warnings[0].category, SyntaxWarning) + def test_fstring_backslash_prefix_raw(self): self.assertEqual(f'\\', '\\') self.assertEqual(f'\\\\', '\\\\') @@ -1632,5 +1660,18 @@ def test_syntax_error_after_debug(self): "f'{1=}{1;}'", ]) + def test_debug_in_file(self): + with temp_cwd(): + script = 'script.py' + with open('script.py', 'w') as f: + f.write(f"""\ +print(f'''{{ +3 +=}}''')""") + + _, stdout, _ = assert_python_ok(script) + self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'), + "3\n=3") + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index d668fa4c3adf5c..c4eca0f5b79511 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -3037,6 +3037,25 @@ def test_access_from_class(self): def test_doc(self): self.assertEqual(CachedCostItem.cost.__doc__, "The cost of the item.") + def test_subclass_with___set__(self): + """Caching still works for a subclass defining __set__.""" + class readonly_cached_property(py_functools.cached_property): + def __set__(self, obj, value): + raise AttributeError("read only property") + + class Test: + def __init__(self, prop): + self._prop = prop + + @readonly_cached_property + def prop(self): + return self._prop + + t = Test(1) + self.assertEqual(t.prop, 1) + t._prop = 999 + self.assertEqual(t.prop, 1) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index b8b591a1bcf2c6..4730bfafbd9cfe 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -4,6 +4,7 @@ import ast import unittest from test.support import import_helper +from test.support.script_helper import spawn_python, kill_python from textwrap import dedent import os import re @@ -121,6 +122,13 @@ def test_unicode_literals_exec(self): exec("from __future__ import unicode_literals; x = ''", {}, scope) self.assertIsInstance(scope["x"], str) + def test_syntactical_future_repl(self): + p = spawn_python('-i') + p.stdin.write(b"from __future__ import barry_as_FLUFL\n") + p.stdin.write(b"2 <> 3\n") + out = kill_python(p) + self.assertNotIn(b'SyntaxError: invalid syntax', out) + class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 24d4216417521c..bf600a0f4d1834 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -209,6 +209,9 @@ class MyList(list): def test_repr(self): class MyList(list): pass + class MyGeneric: + __class_getitem__ = classmethod(GenericAlias) + self.assertEqual(repr(list[str]), 'list[str]') self.assertEqual(repr(list[()]), 'list[()]') self.assertEqual(repr(tuple[int, ...]), 'tuple[int, ...]') @@ -221,6 +224,11 @@ class MyList(list): self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr..MyList[int]')) self.assertEqual(repr(list[str]()), '[]') # instances should keep their normal repr + # gh-105488 + self.assertTrue(repr(MyGeneric[int]).endswith('MyGeneric[int]')) + self.assertTrue(repr(MyGeneric[[]]).endswith('MyGeneric[[]]')) + self.assertTrue(repr(MyGeneric[[int, str]]).endswith('MyGeneric[[int, str]]')) + def test_exposed_type(self): import types a = types.GenericAlias(list, int) diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 6de413e5056ef0..b06b3b09411d62 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -16,6 +16,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure gzip = import_helper.import_module('gzip') +zlib = import_helper.import_module('zlib') data1 = b""" int length=DEFAULTALLOC, err = Z_OK; PyObject *RetVal; @@ -616,6 +617,54 @@ def test_issue44439(self): self.assertEqual(f.write(q), LENGTH) self.assertEqual(f.tell(), LENGTH) + def test_flush_flushes_compressor(self): + # See issue GH-105808. + b = io.BytesIO() + message = b"important message here." + with gzip.GzipFile(fileobj=b, mode='w') as f: + f.write(message) + f.flush() + partial_data = b.getvalue() + full_data = b.getvalue() + self.assertEqual(gzip.decompress(full_data), message) + # The partial data should contain the gzip header and the complete + # message, but not the end-of-stream markers (so we can't just + # decompress it directly). + with self.assertRaises(EOFError): + gzip.decompress(partial_data) + d = zlib.decompressobj(wbits=-zlib.MAX_WBITS) + f = io.BytesIO(partial_data) + gzip._read_gzip_header(f) + read_message = d.decompress(f.read()) + self.assertEqual(read_message, message) + + def test_flush_modes(self): + # Make sure the argument to flush is properly passed to the + # zlib.compressobj; see issue GH-105808. + class FakeCompressor: + def __init__(self): + self.modes = [] + def compress(self, data): + return b'' + def flush(self, mode=-1): + self.modes.append(mode) + return b'' + b = io.BytesIO() + fc = FakeCompressor() + with gzip.GzipFile(fileobj=b, mode='w') as f: + f.compress = fc + f.flush() + f.flush(50) + f.flush(zlib_mode=100) + # The implicit close will also flush the compressor. + expected_modes = [ + zlib.Z_SYNC_FLUSH, + 50, + 100, + -1, + ] + self.assertEqual(fc.modes, expected_modes) + class TestOpen(BaseTest): def test_binary_modes(self): diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 8955d45fa93dd4..fe8105ee2bb3fa 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2404,6 +2404,19 @@ def test_proxy_response_headers(self): headers = self.conn.get_proxy_response_headers() self.assertIn(expected_header, headers.items()) + def test_no_proxy_response_headers(self): + expected_header = ('X-Dummy', '1') + response_text = ( + 'HTTP/1.0 200 OK\r\n' + '{0}\r\n\r\n'.format(':'.join(expected_header)) + ) + + self.conn._create_connection = self._create_connection(response_text) + + self.conn.request('PUT', '/', '') + headers = self.conn.get_proxy_response_headers() + self.assertIsNone(headers) + def test_tunnel_leak(self): sock = None diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index cdd1bea754a020..cfd8a101dcc1c1 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -442,10 +442,10 @@ def test_undecodable_filename(self): def test_undecodable_parameter(self): # sanity check using a valid parameter response = self.request(self.base_url + '/?x=123').read() - self.assertRegex(response, f'listing for {self.base_url}/\?x=123'.encode('latin1')) + self.assertRegex(response, rf'listing for {self.base_url}/\?x=123'.encode('latin1')) # now the bogus encoding response = self.request(self.base_url + '/?x=%bb').read() - self.assertRegex(response, f'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1')) + self.assertRegex(response, rf'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1')) def test_get_dir_redirect_location_domain_injection_bug(self): """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 227c912bc8c322..ee9fe4d574dd42 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -23,6 +23,7 @@ import unittest from unittest import mock import _testinternalcapi +import _imp from test.support import os_helper from test.support import ( @@ -763,6 +764,13 @@ def test_dll_dependency_import(self): env=env, cwd=os.path.dirname(pyexe)) + def test_issue105979(self): + # this used to crash + with self.assertRaises(ImportError) as cm: + _imp.get_frozen_object("x", b"6\'\xd5Cu\x12") + self.assertIn("Frozen object named 'x' is invalid", + str(cm.exception)) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): @@ -1640,9 +1648,10 @@ class SubinterpImportTests(unittest.TestCase): ) ISOLATED = dict( use_main_obmalloc=False, - own_gil=True, + gil=2, ) NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()} + NOT_ISOLATED['gil'] = 1 @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def pipe(self): diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py index 46cd2b696d4cc8..81f683799cbc80 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/test_main.py @@ -69,7 +69,7 @@ def test_abc_enforced(self): dict(name=''), ) def test_invalid_inputs_to_from_name(self, name): - with self.assertRaises(Exception): + with self.assertRaises(ValueError): Distribution.from_name(name) diff --git a/Lib/test/test_json/__init__.py b/Lib/test/test_json/__init__.py index 37b2e0d5e26d16..74b64ed86a3183 100644 --- a/Lib/test/test_json/__init__.py +++ b/Lib/test/test_json/__init__.py @@ -18,7 +18,6 @@ class PyTest(unittest.TestCase): json = pyjson loads = staticmethod(pyjson.loads) dumps = staticmethod(pyjson.dumps) - AttrDict = pyjson.AttrDict JSONDecodeError = staticmethod(pyjson.JSONDecodeError) @unittest.skipUnless(cjson, 'requires _json') diff --git a/Lib/test/test_json/test_attrdict.py b/Lib/test/test_json/test_attrdict.py deleted file mode 100644 index 143ea462d310aa..00000000000000 --- a/Lib/test/test_json/test_attrdict.py +++ /dev/null @@ -1,145 +0,0 @@ -from test.test_json import PyTest -import pickle -import sys -import unittest - -kepler_dict = { - "orbital_period": { - "mercury": 88, - "venus": 225, - "earth": 365, - "mars": 687, - "jupiter": 4331, - "saturn": 10_756, - "uranus": 30_687, - "neptune": 60_190, - }, - "dist_from_sun": { - "mercury": 58, - "venus": 108, - "earth": 150, - "mars": 228, - "jupiter": 778, - "saturn": 1_400, - "uranus": 2_900, - "neptune": 4_500, - } -} - -class TestAttrDict(PyTest): - - def test_dict_subclass(self): - self.assertTrue(issubclass(self.AttrDict, dict)) - - def test_slots(self): - d = self.AttrDict(x=1, y=2) - with self.assertRaises(TypeError): - vars(d) - - def test_constructor_signatures(self): - AttrDict = self.AttrDict - target = dict(x=1, y=2) - self.assertEqual(AttrDict(x=1, y=2), target) # kwargs - self.assertEqual(AttrDict(dict(x=1, y=2)), target) # mapping - self.assertEqual(AttrDict(dict(x=1, y=0), y=2), target) # mapping, kwargs - self.assertEqual(AttrDict([('x', 1), ('y', 2)]), target) # iterable - self.assertEqual(AttrDict([('x', 1), ('y', 0)], y=2), target) # iterable, kwargs - - def test_getattr(self): - d = self.AttrDict(x=1, y=2) - self.assertEqual(d.x, 1) - with self.assertRaises(AttributeError): - d.z - - def test_setattr(self): - d = self.AttrDict(x=1, y=2) - d.x = 3 - d.z = 5 - self.assertEqual(d, dict(x=3, y=2, z=5)) - - def test_delattr(self): - d = self.AttrDict(x=1, y=2) - del d.x - self.assertEqual(d, dict(y=2)) - with self.assertRaises(AttributeError): - del d.z - - def test_dir(self): - d = self.AttrDict(x=1, y=2) - self.assertTrue(set(dir(d)), set(dir(dict)).union({'x', 'y'})) - - def test_repr(self): - # This repr is doesn't round-trip. It matches a regular dict. - # That seems to be the norm for AttrDict recipes being used - # in the wild. Also it supports the design concept that an - # AttrDict is just like a regular dict but has optional - # attribute style lookup. - self.assertEqual(repr(self.AttrDict(x=1, y=2)), - repr(dict(x=1, y=2))) - - def test_overlapping_keys_and_methods(self): - d = self.AttrDict(items=50) - self.assertEqual(d['items'], 50) - self.assertEqual(d.items(), dict(d).items()) - - def test_invalid_attribute_names(self): - d = self.AttrDict({ - 'control': 'normal case', - 'class': 'keyword', - 'two words': 'contains space', - 'hypen-ate': 'contains a hyphen' - }) - self.assertEqual(d.control, dict(d)['control']) - self.assertEqual(d['class'], dict(d)['class']) - self.assertEqual(d['two words'], dict(d)['two words']) - self.assertEqual(d['hypen-ate'], dict(d)['hypen-ate']) - - def test_object_hook_use_case(self): - AttrDict = self.AttrDict - json_string = self.dumps(kepler_dict) - kepler_ad = self.loads(json_string, object_hook=AttrDict) - - self.assertEqual(kepler_ad, kepler_dict) # Match regular dict - self.assertIsInstance(kepler_ad, AttrDict) # Verify conversion - self.assertIsInstance(kepler_ad.orbital_period, AttrDict) # Nested - - # Exercise dotted lookups - self.assertEqual(kepler_ad.orbital_period, kepler_dict['orbital_period']) - self.assertEqual(kepler_ad.orbital_period.earth, - kepler_dict['orbital_period']['earth']) - self.assertEqual(kepler_ad['orbital_period'].earth, - kepler_dict['orbital_period']['earth']) - - # Dict style error handling and Attribute style error handling - with self.assertRaises(KeyError): - kepler_ad.orbital_period['pluto'] - with self.assertRaises(AttributeError): - kepler_ad.orbital_period.Pluto - - # Order preservation - self.assertEqual(list(kepler_ad.items()), list(kepler_dict.items())) - self.assertEqual(list(kepler_ad.orbital_period.items()), - list(kepler_dict['orbital_period'].items())) - - # Round trip - self.assertEqual(self.dumps(kepler_ad), json_string) - - def test_pickle(self): - AttrDict = self.AttrDict - json_string = self.dumps(kepler_dict) - kepler_ad = self.loads(json_string, object_hook=AttrDict) - - # Pickling requires the cached module to be the real module - cached_module = sys.modules.get('json') - sys.modules['json'] = self.json - try: - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - kepler_ad2 = pickle.loads(pickle.dumps(kepler_ad, protocol)) - self.assertEqual(kepler_ad2, kepler_ad) - self.assertEqual(type(kepler_ad2), AttrDict) - finally: - sys.modules['json'] = cached_module - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index c2cf058c321fa5..9f28ced32bd26c 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -539,6 +539,28 @@ def b(): self._check_in_scopes(code, {"x": True, "y": ["b"]}, scopes=["function"]) self._check_in_scopes(code, raises=NameError, scopes=["class"]) + def test_iter_var_available_in_locals(self): + code = """ + l = [1, 2] + y = 0 + items = [locals()["x"] for x in l] + items2 = [vars()["x"] for x in l] + items3 = [("x" in dir()) for x in l] + items4 = [eval("x") for x in l] + # x is available, and does not overwrite y + [exec("y = x") for x in l] + """ + self._check_in_scopes( + code, + { + "items": [1, 2], + "items2": [1, 2], + "items3": [True, True], + "items4": [1, 2], + "y": 0 + } + ) + __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 4c592eaf34da23..4977a9369ddf88 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -116,10 +116,13 @@ def test_add_nonascii_string_header_raises(self): self.assertMailboxEmpty() def test_add_that_raises_leaves_mailbox_empty(self): + class CustomError(Exception): ... + exc_msg = "a fake error" + def raiser(*args, **kw): - raise Exception("a fake error") + raise CustomError(exc_msg) support.patch(self, email.generator.BytesGenerator, 'flatten', raiser) - with self.assertRaises(Exception): + with self.assertRaisesRegex(CustomError, exc_msg): self._box.add(email.message_from_string("From: Alphöso")) self.assertEqual(len(self._box), 0) self._box.close() diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 517cbe0cb115ab..bab868600895c1 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -299,6 +299,27 @@ def test_find_end(self): self.assertEqual(m.find(b'one', 1, -2), -1) self.assertEqual(m.find(bytearray(b'one')), 0) + for i in range(-n-1, n+1): + for j in range(-n-1, n+1): + for p in [b"o", b"on", b"two", b"ones", b"s"]: + expected = data.find(p, i, j) + self.assertEqual(m.find(p, i, j), expected, (p, i, j)) + + def test_find_does_not_access_beyond_buffer(self): + try: + flags = mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS + PAGESIZE = mmap.PAGESIZE + PROT_NONE = 0 + PROT_READ = mmap.PROT_READ + except AttributeError as e: + raise unittest.SkipTest("mmap flags unavailable") from e + for i in range(0, 2049): + with mmap.mmap(-1, PAGESIZE * (i + 1), + flags=flags, prot=PROT_NONE) as guard: + with mmap.mmap(-1, PAGESIZE * (i + 2048), + flags=flags, prot=PROT_READ) as fm: + fm.find(b"fo", -2) + def test_rfind(self): # test the new 'end' parameter works as expected diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 57fed5d09fd7b8..5281eb77c02d1b 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -452,6 +452,35 @@ def f(): self.assertFalse(f()) +class TestCallCache(unittest.TestCase): + def test_too_many_defaults_0(self): + def f(): + pass + + f.__defaults__ = (None,) + for _ in range(1025): + f() + + def test_too_many_defaults_1(self): + def f(x): + pass + + f.__defaults__ = (None, None) + for _ in range(1025): + f(None) + f() + + def test_too_many_defaults_2(self): + def f(x, y): + pass + + f.__defaults__ = (None, None, None) + for _ in range(1025): + f(None, None) + f(None) + f() + + if __name__ == "__main__": import unittest unittest.main() diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index bf4decf9f97ae8..1d28a782f44ab3 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -317,6 +317,10 @@ def test_match_common(self): self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) + # Matching against empty path + self.assertFalse(P().match('*')) + self.assertTrue(P().match('**')) + self.assertFalse(P().match('**/*')) def test_ordering_common(self): # Ordering is tuple-alike. @@ -1985,7 +1989,7 @@ def test_glob_long_symlink(self): self.assertEqual(sorted(base.glob('**/*')), [bad_link]) def test_glob_above_recursion_limit(self): - recursion_limit = 40 + recursion_limit = 50 # directory_depth > recursion_limit directory_depth = recursion_limit + 10 base = pathlib.Path(os_helper.TESTFN, 'deep') diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py index 7c402c3d7c5acf..77f72fcc7c6e3b 100644 --- a/Lib/test/test_peg_generator/__init__.py +++ b/Lib/test/test_peg_generator/__init__.py @@ -3,9 +3,6 @@ from test import support from test.support import load_package_tests -# TODO: gh-92584: peg_generator uses distutils which was removed in Python 3.12 -raise unittest.SkipTest("distutils has been removed in Python 3.12") - if support.check_sanitizer(address=True, memory=True): # bpo-46633: Skip the test because it is too slow when Python is built diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index d34ffef0dbc5ec..f9105a9f23bd6d 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -1,3 +1,5 @@ +import contextlib +import subprocess import sysconfig import textwrap import unittest @@ -8,7 +10,7 @@ from test import test_tools from test import support -from test.support import os_helper +from test.support import os_helper, import_helper from test.support.script_helper import assert_python_ok _py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST") @@ -72,8 +74,18 @@ def test_parse(self): @support.requires_subprocess() class TestCParser(unittest.TestCase): + _has_run = False + @classmethod def setUpClass(cls): + if cls._has_run: + # Since gh-104798 (Use setuptools in peg-generator and reenable + # tests), this test case has been producing ref leaks. Initial + # debugging points to bug(s) in setuptools and/or importlib. + # See gh-105063 for more info. + raise unittest.SkipTest("gh-105063: can not rerun because of ref. leaks") + cls._has_run = True + # When running under regtest, a separate tempdir is used # as the current directory and watched for left-overs. # Reusing that as the base for temporary directories @@ -88,6 +100,16 @@ def setUpClass(cls): cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base) cls.addClassCleanup(shutil.rmtree, cls.library_dir) + with contextlib.ExitStack() as stack: + python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv")) + sitepackages = subprocess.check_output( + [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"], + text=True, + ).strip() + stack.enter_context(import_helper.DirsOnSysPath(sitepackages)) + cls.addClassCleanup(stack.pop_all().close) + + @support.requires_venv_with_pip() def setUp(self): self._backup_config_vars = dict(sysconfig._CONFIG_VARS) cmd = support.missing_compiler_executable() diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 30e992ed213c67..d92da7b29bff98 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -794,7 +794,7 @@ def test_soft_keyword(self) -> None: start: | "number" n=NUMBER { eval(n.string) } | "string" n=STRING { n.string } - | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { f"{l.string} = {n.string}"} + | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { l.string + " = " + n.string } """ parser_class = make_parser(grammar) self.assertEqual(parse_string("number 1", parser_class), 1) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index cefc71cb5a7f54..ddb5187f90da9b 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -24,7 +24,8 @@ from urllib.request import urlopen, urlcleanup from test.support import import_helper from test.support import os_helper -from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support.script_helper import (assert_python_ok, + assert_python_failure, spawn_python) from test.support import threading_helper from test.support import (reap_children, captured_output, captured_stdout, captured_stderr, is_emscripten, is_wasi, @@ -631,6 +632,21 @@ def test_builtin_on_metaclasses(self): # Testing that the subclasses section does not appear self.assertNotIn('Built-in subclasses', text) + def test_fail_help_cli(self): + elines = (missing_pattern % 'abd').splitlines() + with spawn_python("-c" "help()") as proc: + out, _ = proc.communicate(b"abd") + olines = out.decode().splitlines()[-9:-6] + olines[0] = olines[0].removeprefix('help> ') + self.assertEqual(elines, olines) + + def test_fail_help_output_redirect(self): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.help("abd") + expected = missing_pattern % "abd" + self.assertEqual(expected, buf.getvalue().strip().replace('\n', os.linesep)) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') @requires_docstrings diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 11628a236ade9a..50b9ad701f0ce7 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1046,33 +1046,6 @@ def test_ignore_case_range(self): def test_category(self): self.assertEqual(re.match(r"(\s)", " ").group(1), " ") - @cpython_only - def test_case_helpers(self): - import _sre - for i in range(128): - c = chr(i) - lo = ord(c.lower()) - self.assertEqual(_sre.ascii_tolower(i), lo) - self.assertEqual(_sre.unicode_tolower(i), lo) - iscased = c in string.ascii_letters - self.assertEqual(_sre.ascii_iscased(i), iscased) - self.assertEqual(_sre.unicode_iscased(i), iscased) - - for i in list(range(128, 0x1000)) + [0x10400, 0x10428]: - c = chr(i) - self.assertEqual(_sre.ascii_tolower(i), i) - if i != 0x0130: - self.assertEqual(_sre.unicode_tolower(i), ord(c.lower())) - iscased = c != c.lower() or c != c.upper() - self.assertFalse(_sre.ascii_iscased(i)) - self.assertEqual(_sre.unicode_iscased(i), - c != c.lower() or c != c.upper()) - - self.assertEqual(_sre.ascii_tolower(0x0130), 0x0130) - self.assertEqual(_sre.unicode_tolower(0x0130), ord('i')) - self.assertFalse(_sre.ascii_iscased(0x0130)) - self.assertTrue(_sre.unicode_iscased(0x0130)) - def test_not_literal(self): self.assertEqual(re.search(r"\s([^a])", " b").group(1), "b") self.assertEqual(re.search(r"\s([^a]*)", " bb").group(1), "bb") @@ -1769,20 +1742,6 @@ def test_bug_6509(self): pat = re.compile(b'..') self.assertEqual(pat.sub(lambda m: b'bytes', b'a5'), b'bytes') - def test_dealloc(self): - # issue 3299: check for segfault in debug build - import _sre - # the overflow limit is different on wide and narrow builds and it - # depends on the definition of SRE_CODE (see sre.h). - # 2**128 should be big enough to overflow on both. For smaller values - # a RuntimeError is raised instead of OverflowError. - long_overflow = 2**128 - self.assertRaises(TypeError, re.finditer, "a", {}) - with self.assertRaises(OverflowError): - _sre.compile("abc", 0, [long_overflow], 0, {}, ()) - with self.assertRaises(TypeError): - _sre.compile({}, 0, [], 0, [], []) - def test_search_dot_unicode(self): self.assertTrue(re.search("123.*-", '123abc-')) self.assertTrue(re.search("123.*-", '123\xe9-')) @@ -1840,21 +1799,6 @@ def test_repeat_minmax_overflow(self): self.assertRaises(OverflowError, re.compile, r".{%d,}?" % 2**128) self.assertRaises(OverflowError, re.compile, r".{%d,%d}" % (2**129, 2**128)) - @cpython_only - def test_repeat_minmax_overflow_maxrepeat(self): - try: - from _sre import MAXREPEAT - except ImportError: - self.skipTest('requires _sre.MAXREPEAT constant') - string = "x" * 100000 - self.assertIsNone(re.match(r".{%d}" % (MAXREPEAT - 1), string)) - self.assertEqual(re.match(r".{,%d}" % (MAXREPEAT - 1), string).span(), - (0, 100000)) - self.assertIsNone(re.match(r".{%d,}?" % (MAXREPEAT - 1), string)) - self.assertRaises(OverflowError, re.compile, r".{%d}" % MAXREPEAT) - self.assertRaises(OverflowError, re.compile, r".{,%d}" % MAXREPEAT) - self.assertRaises(OverflowError, re.compile, r".{%d,}?" % MAXREPEAT) - def test_backref_group_name_in_exception(self): # Issue 17341: Poor error message when compiling invalid regex self.checkPatternError('(?P=)', @@ -2502,7 +2446,10 @@ def test_debug_flag(self): def test_atomic_group(self): self.assertEqual(get_debug_out(r'(?>ab?)'), '''\ -ATOMIC_GROUP [(LITERAL, 97), (MAX_REPEAT, (0, 1, [(LITERAL, 98)]))] +ATOMIC_GROUP + LITERAL 97 + MAX_REPEAT 0 1 + LITERAL 98 0. INFO 4 0b0 1 2 (to 5) 5: ATOMIC_GROUP 11 (to 17) @@ -2686,6 +2633,75 @@ def test_deprecated_modules(self): self.assertTrue(hasattr(mod, attr)) del sys.modules[name] + @cpython_only + def test_case_helpers(self): + import _sre + for i in range(128): + c = chr(i) + lo = ord(c.lower()) + self.assertEqual(_sre.ascii_tolower(i), lo) + self.assertEqual(_sre.unicode_tolower(i), lo) + iscased = c in string.ascii_letters + self.assertEqual(_sre.ascii_iscased(i), iscased) + self.assertEqual(_sre.unicode_iscased(i), iscased) + + for i in list(range(128, 0x1000)) + [0x10400, 0x10428]: + c = chr(i) + self.assertEqual(_sre.ascii_tolower(i), i) + if i != 0x0130: + self.assertEqual(_sre.unicode_tolower(i), ord(c.lower())) + iscased = c != c.lower() or c != c.upper() + self.assertFalse(_sre.ascii_iscased(i)) + self.assertEqual(_sre.unicode_iscased(i), + c != c.lower() or c != c.upper()) + + self.assertEqual(_sre.ascii_tolower(0x0130), 0x0130) + self.assertEqual(_sre.unicode_tolower(0x0130), ord('i')) + self.assertFalse(_sre.ascii_iscased(0x0130)) + self.assertTrue(_sre.unicode_iscased(0x0130)) + + @cpython_only + def test_dealloc(self): + # issue 3299: check for segfault in debug build + import _sre + # the overflow limit is different on wide and narrow builds and it + # depends on the definition of SRE_CODE (see sre.h). + # 2**128 should be big enough to overflow on both. For smaller values + # a RuntimeError is raised instead of OverflowError. + long_overflow = 2**128 + self.assertRaises(TypeError, re.finditer, "a", {}) + with self.assertRaises(OverflowError): + _sre.compile("abc", 0, [long_overflow], 0, {}, ()) + with self.assertRaises(TypeError): + _sre.compile({}, 0, [], 0, [], []) + + @cpython_only + def test_repeat_minmax_overflow_maxrepeat(self): + try: + from _sre import MAXREPEAT + except ImportError: + self.skipTest('requires _sre.MAXREPEAT constant') + string = "x" * 100000 + self.assertIsNone(re.match(r".{%d}" % (MAXREPEAT - 1), string)) + self.assertEqual(re.match(r".{,%d}" % (MAXREPEAT - 1), string).span(), + (0, 100000)) + self.assertIsNone(re.match(r".{%d,}?" % (MAXREPEAT - 1), string)) + self.assertRaises(OverflowError, re.compile, r".{%d}" % MAXREPEAT) + self.assertRaises(OverflowError, re.compile, r".{,%d}" % MAXREPEAT) + self.assertRaises(OverflowError, re.compile, r".{%d,}?" % MAXREPEAT) + + @cpython_only + def test_sre_template_invalid_group_index(self): + # see gh-106524 + import _sre + with self.assertRaises(TypeError) as cm: + _sre.template("", ["", -1, ""]) + self.assertIn("invalid template", str(cm.exception)) + with self.assertRaises(TypeError) as cm: + _sre.template("", ["", (), ""]) + self.assertIn("an integer is required", str(cm.exception)) + + class ExternalTests(unittest.TestCase): def test_re_benchmarks(self): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ac49fbae847726..806b932a164df8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,6 +7,7 @@ import contextlib import glob import io +import locale import os.path import platform import re @@ -1551,6 +1552,41 @@ def test_leak_tmp_file(self): f"files (1): mytmpfile", output) + def test_mp_decode_error(self): + # gh-101634: If a worker stdout cannot be decoded, report a failed test + # and a non-zero exit code. + if sys.platform == 'win32': + encoding = locale.getencoding() + else: + encoding = sys.stdout.encoding + if encoding is None: + encoding = sys.__stdout__.encoding + if encoding is None: + self.skipTest(f"cannot get regrtest worker encoding") + + nonascii = b"byte:\xa0\xa9\xff\n" + try: + nonascii.decode(encoding) + except UnicodeDecodeError: + pass + else: + self.skipTest(f"{encoding} can decode non-ASCII bytes {nonascii!a}") + + code = textwrap.dedent(fr""" + import sys + # bytes which cannot be decoded from UTF-8 + nonascii = {nonascii!a} + sys.stdout.buffer.write(nonascii) + sys.stdout.buffer.flush() + """) + testname = self.create_test(code=code) + + output = self.run_tests("--fail-env-changed", "-v", "-j1", testname, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], + failed=[testname], + randomize=True) + class TestUtils(unittest.TestCase): def test_format_duration(self): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 36f0b8a31a3715..d74eee767c991d 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2739,7 +2739,7 @@ def test_regular_copy(self): def test_same_file(self): self.addCleanup(self.reset) with self.get_files() as (src, dst): - with self.assertRaises(Exception): + with self.assertRaises((OSError, _GiveupOnFastCopy)): self.zerocopy_fun(src, src) # Make sure src file is not corrupted. self.assertEqual(read_file(TESTFN, binary=True), self.FILEDATA) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index d46ce5e60e2141..6117ca3fdba1b7 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -339,6 +339,15 @@ def test_constants(self): ssl.OP_NO_TLSv1_2 self.assertEqual(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv23) + def test_options(self): + # gh-106687: SSL options values are unsigned integer (uint64_t) + for name in dir(ssl): + if not name.startswith('OP_'): + continue + with self.subTest(option=name): + value = getattr(ssl, name) + self.assertGreaterEqual(value, 0, f"ssl.{name}") + def test_ssl_types(self): ssl_types = [ _ssl._SSLContext, @@ -951,6 +960,7 @@ def test_get_ciphers(self): ) def test_options(self): + # Test default SSLContext options ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) @@ -959,16 +969,30 @@ def test_options(self): OP_SINGLE_DH_USE | OP_SINGLE_ECDH_USE | OP_ENABLE_MIDDLEBOX_COMPAT) self.assertEqual(default, ctx.options) + + # disallow TLSv1 with warnings_helper.check_warnings(): ctx.options |= ssl.OP_NO_TLSv1 self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) + + # allow TLSv1 with warnings_helper.check_warnings(): ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1) self.assertEqual(default, ctx.options) + + # clear all options ctx.options = 0 # Ubuntu has OP_NO_SSLv3 forced on by default self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3) + # invalid options + with self.assertRaises(OverflowError): + ctx.options = -1 + with self.assertRaises(OverflowError): + ctx.options = 2 ** 100 + with self.assertRaises(TypeError): + ctx.options = "abc" + def test_verify_mode_protocol(self): with warnings_helper.check_warnings(): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 51ba423a0f1c92..817eab0c8a7e19 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1692,6 +1692,14 @@ def test_run_with_pathlike_path_and_arguments(self): res = subprocess.run(args) self.assertEqual(res.returncode, 57) + @unittest.skipUnless(mswindows, "Maybe test trigger a leak on Ubuntu") + def test_run_with_an_empty_env(self): + # gh-105436: fix subprocess.run(..., env={}) broken on Windows + args = [sys.executable, "-c", 'pass'] + # Ignore subprocess errors - we only care that the API doesn't + # raise an OSError + subprocess.run(args, env={}) + def test_capture_output(self): cp = self.run_python(("import sys;" "sys.stdout.write('BDFL'); " diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 477879db2fd493..f3d6cd7bad0eec 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1621,6 +1621,22 @@ Traceback (most recent call last): SyntaxError: Did you mean to use 'from ... import ...' instead? +>>> import a, b,c from b +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z, b.y.z, c.y.z from b.y.z +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a,b,c from b as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z, b.y.z, c.y.z from b.y.z as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + # Check that we dont raise the "trailing comma" error if there is more # input to the left of the valid part that we parsed. diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index df9c9db322dc94..5ad278499c9df9 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -558,6 +558,19 @@ def test_string(self): OP '}' (1, 39) (1, 40) FSTRING_MIDDLE ' final words' (1, 40) (1, 52) FSTRING_END "'" (1, 52) (1, 53) + """) + self.check_tokenize("""\ +f'''{ +3 +=}'''""", """\ + FSTRING_START "f'''" (1, 0) (1, 4) + OP '{' (1, 4) (1, 5) + NL '\\n' (1, 5) (1, 6) + NUMBER '3' (2, 0) (2, 1) + NL '\\n' (2, 1) (2, 2) + OP '=' (3, 0) (3, 1) + OP '}' (3, 1) (3, 2) + FSTRING_END "'''" (3, 2) (3, 5) """) def test_function(self): @@ -2239,6 +2252,16 @@ def test_string(self): FSTRING_START \'f"\' (1, 0) (1, 2) FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 2) (2, 3) FSTRING_END '"' (2, 3) (2, 4) + """) + + self.check_tokenize('''\ +f"{ +a}"''', """\ + FSTRING_START 'f"' (1, 0) (1, 2) + LBRACE '{' (1, 2) (1, 3) + NAME 'a' (2, 0) (2, 1) + RBRACE '}' (2, 1) (2, 2) + FSTRING_END '"' (2, 2) (2, 3) """) self.check_tokenize(r'Rf"abc\ diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index a3067e521e023e..0ce97f57de6860 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -142,7 +142,16 @@ def test_subscripting(self): def test_repr(self): type Simple = int + type VeryGeneric[T, *Ts, **P] = Callable[P, tuple[T, *Ts]] + self.assertEqual(repr(Simple), "Simple") + self.assertEqual(repr(VeryGeneric), "VeryGeneric") + self.assertEqual(repr(VeryGeneric[int, bytes, str, [float, object]]), + "VeryGeneric[int, bytes, str, [float, object]]") + self.assertEqual(repr(VeryGeneric[int, []]), + "VeryGeneric[int, []]") + self.assertEqual(repr(VeryGeneric[int, [VeryGeneric[int], list[str]]]), + "VeryGeneric[int, [VeryGeneric[int], list[str]]]") def test_recursive_repr(self): type Recursive = Recursive @@ -151,6 +160,13 @@ def test_recursive_repr(self): type X = list[Y] type Y = list[X] self.assertEqual(repr(X), "X") + self.assertEqual(repr(Y), "Y") + + type GenericRecursive[X] = list[X | GenericRecursive[X]] + self.assertEqual(repr(GenericRecursive), "GenericRecursive") + self.assertEqual(repr(GenericRecursive[int]), "GenericRecursive[int]") + self.assertEqual(repr(GenericRecursive[GenericRecursive[int]]), + "GenericRecursive[GenericRecursive[int]]") class TypeAliasConstructorTest(unittest.TestCase): @@ -228,8 +244,69 @@ def test_module(self): self.assertEqual(mod_generics_cache.OldStyle.__module__, mod_generics_cache.__name__) + +# All these type aliases are used for pickling tests: +T = TypeVar('T') +type SimpleAlias = int +type RecursiveAlias = dict[str, RecursiveAlias] +type GenericAlias[X] = list[X] +type GenericAliasMultipleTypes[X, Y] = dict[X, Y] +type RecursiveGenericAlias[X] = dict[str, RecursiveAlias[X]] +type BoundGenericAlias[X: int] = set[X] +type ConstrainedGenericAlias[LongName: (str, bytes)] = list[LongName] +type AllTypesAlias[A, *B, **C] = Callable[C, A] | tuple[*B] + + +class TypeAliasPickleTest(unittest.TestCase): def test_pickling(self): - pickled = pickle.dumps(mod_generics_cache.Alias) - self.assertIs(pickle.loads(pickled), mod_generics_cache.Alias) - pickled = pickle.dumps(mod_generics_cache.OldStyle) - self.assertIs(pickle.loads(pickled), mod_generics_cache.OldStyle) + things_to_test = [ + SimpleAlias, + RecursiveAlias, + + GenericAlias, + GenericAlias[T], + GenericAlias[int], + + GenericAliasMultipleTypes, + GenericAliasMultipleTypes[str, T], + GenericAliasMultipleTypes[T, str], + GenericAliasMultipleTypes[int, str], + + RecursiveGenericAlias, + RecursiveGenericAlias[T], + RecursiveGenericAlias[int], + + BoundGenericAlias, + BoundGenericAlias[int], + BoundGenericAlias[T], + + ConstrainedGenericAlias, + ConstrainedGenericAlias[str], + ConstrainedGenericAlias[T], + + AllTypesAlias, + AllTypesAlias[int, str, T, [T, object]], + + # Other modules: + mod_generics_cache.Alias, + mod_generics_cache.OldStyle, + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + self.assertEqual(pickle.loads(pickled), thing) + + type ClassLevel = str + + def test_pickling_local(self): + type A = int + things_to_test = [ + self.ClassLevel, + A, + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + with self.assertRaises(pickle.PickleError): + pickle.dumps(thing, protocol=proto) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 6475df6f5bba43..bced641a9661fd 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -2,6 +2,8 @@ import textwrap import types import unittest +import pickle +import weakref from test.support import requires_working_socket, check_syntax_error, run_code from typing import Generic, Sequence, TypeVar, TypeVarTuple, ParamSpec, get_args @@ -855,3 +857,98 @@ def func[A](): ns = run_code(code) self.assertEqual(ns["func"].__type_params__, ()) + + + +# All these type aliases are used for pickling tests: +T = TypeVar('T') +def func1[X](x: X) -> X: ... +def func2[X, Y](x: X | Y) -> X | Y: ... +def func3[X, *Y, **Z](x: X, y: tuple[*Y], z: Z) -> X: ... +def func4[X: int, Y: (bytes, str)](x: X, y: Y) -> X | Y: ... + +class Class1[X]: ... +class Class2[X, Y]: ... +class Class3[X, *Y, **Z]: ... +class Class4[X: int, Y: (bytes, str)]: ... + + +class TypeParamsPickleTest(unittest.TestCase): + def test_pickling_functions(self): + things_to_test = [ + func1, + func2, + func3, + func4, + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + self.assertEqual(pickle.loads(pickled), thing) + + def test_pickling_classes(self): + things_to_test = [ + Class1, + Class1[int], + Class1[T], + + Class2, + Class2[int, T], + Class2[T, int], + Class2[int, str], + + Class3, + Class3[int, T, str, bytes, [float, object, T]], + + Class4, + Class4[int, bytes], + Class4[T, bytes], + Class4[int, T], + Class4[T, T], + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + self.assertEqual(pickle.loads(pickled), thing) + + for klass in things_to_test: + real_class = getattr(klass, '__origin__', klass) + thing = klass() + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + # These instances are not equal, + # but class check is good enough: + self.assertIsInstance(pickle.loads(pickled), real_class) + + +class TypeParamsWeakRefTest(unittest.TestCase): + def test_weakrefs(self): + T = TypeVar('T') + P = ParamSpec('P') + class OldStyle(Generic[T]): + pass + + class NewStyle[T]: + pass + + cases = [ + T, + TypeVar('T', bound=int), + P, + P.args, + P.kwargs, + TypeVarTuple('Ts'), + OldStyle, + OldStyle[int], + OldStyle(), + NewStyle, + NewStyle[int], + NewStyle(), + Generic[T], + ] + for case in cases: + with self.subTest(case=case): + weakref.ref(case) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 7dc64c6ecc9e5b..c88152de7725fb 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2759,6 +2759,80 @@ def x(self): ... with self.assertRaisesRegex(TypeError, only_classes_allowed): issubclass(1, BadPG) + def test_implicit_issubclass_between_two_protocols(self): + @runtime_checkable + class CallableMembersProto(Protocol): + def meth(self): ... + + # All the below protocols should be considered "subclasses" + # of CallableMembersProto at runtime, + # even though none of them explicitly subclass CallableMembersProto + + class IdenticalProto(Protocol): + def meth(self): ... + + class SupersetProto(Protocol): + def meth(self): ... + def meth2(self): ... + + class NonCallableMembersProto(Protocol): + meth: Callable[[], None] + + class NonCallableMembersSupersetProto(Protocol): + meth: Callable[[], None] + meth2: Callable[[str, int], bool] + + class MixedMembersProto1(Protocol): + meth: Callable[[], None] + def meth2(self): ... + + class MixedMembersProto2(Protocol): + def meth(self): ... + meth2: Callable[[str, int], bool] + + for proto in ( + IdenticalProto, SupersetProto, NonCallableMembersProto, + NonCallableMembersSupersetProto, MixedMembersProto1, MixedMembersProto2 + ): + with self.subTest(proto=proto.__name__): + self.assertIsSubclass(proto, CallableMembersProto) + + # These two shouldn't be considered subclasses of CallableMembersProto, however, + # since they don't have the `meth` protocol member + + class EmptyProtocol(Protocol): ... + class UnrelatedProtocol(Protocol): + def wut(self): ... + + self.assertNotIsSubclass(EmptyProtocol, CallableMembersProto) + self.assertNotIsSubclass(UnrelatedProtocol, CallableMembersProto) + + # These aren't protocols at all (despite having annotations), + # so they should only be considered subclasses of CallableMembersProto + # if they *actually have an attribute* matching the `meth` member + # (just having an annotation is insufficient) + + class AnnotatedButNotAProtocol: + meth: Callable[[], None] + + class NotAProtocolButAnImplicitSubclass: + def meth(self): pass + + class NotAProtocolButAnImplicitSubclass2: + meth: Callable[[], None] + def meth(self): pass + + class NotAProtocolButAnImplicitSubclass3: + meth: Callable[[], None] + meth2: Callable[[int, str], bool] + def meth(self): pass + def meth(self, x, y): return True + + self.assertNotIsSubclass(AnnotatedButNotAProtocol, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass2, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass3, CallableMembersProto) + def test_isinstance_checks_not_at_whim_of_gc(self): self.addCleanup(gc.enable) gc.disable() @@ -3391,6 +3465,46 @@ def __subclasshook__(cls, other): self.assertIsSubclass(OKClass, C) self.assertNotIsSubclass(BadClass, C) + def test_custom_subclasshook_2(self): + @runtime_checkable + class HasX(Protocol): + # The presence of a non-callable member + # would mean issubclass() checks would fail with TypeError + # if it weren't for the custom `__subclasshook__` method + x = 1 + + @classmethod + def __subclasshook__(cls, other): + return hasattr(other, 'x') + + class Empty: pass + + class ImplementsHasX: + x = 1 + + self.assertIsInstance(ImplementsHasX(), HasX) + self.assertNotIsInstance(Empty(), HasX) + self.assertIsSubclass(ImplementsHasX, HasX) + self.assertNotIsSubclass(Empty, HasX) + + # isinstance() and issubclass() checks against this still raise TypeError, + # despite the presence of the custom __subclasshook__ method, + # as it's not decorated with @runtime_checkable + class NotRuntimeCheckable(Protocol): + @classmethod + def __subclasshook__(cls, other): + return hasattr(other, 'x') + + must_be_runtime_checkable = ( + "Instance and class checks can only be used " + "with @runtime_checkable protocols" + ) + + with self.assertRaisesRegex(TypeError, must_be_runtime_checkable): + issubclass(object, NotRuntimeCheckable) + with self.assertRaisesRegex(TypeError, must_be_runtime_checkable): + isinstance(object(), NotRuntimeCheckable) + def test_issubclass_fails_correctly(self): @runtime_checkable class P(Protocol): diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 5f12f9f956674a..edd9a5dfc1f89d 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -436,9 +436,10 @@ async def addition(self, var): pass self.assertEqual(output, 10) async def test_add_side_effect_exception(self): + class CustomError(Exception): pass async def addition(var): pass - mock = AsyncMock(addition, side_effect=Exception('err')) - with self.assertRaises(Exception): + mock = AsyncMock(addition, side_effect=CustomError('side-effect')) + with self.assertRaisesRegex(CustomError, 'side-effect'): await mock(5) async def test_add_side_effect_coroutine(self): diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 88c7c3a0af8771..b3efb61e83049e 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -1,4 +1,4 @@ -"""Tests for the unparse.py script in the Tools/parser directory.""" +"""Tests for ast.unparse.""" import unittest import test.support @@ -625,6 +625,78 @@ def test_star_expr_assign_target_multiple(self): self.check_src_roundtrip("a, b = [c, d] = e, f = g") +class ManualASTCreationTestCase(unittest.TestCase): + """Test that AST nodes created without a type_params field unparse correctly.""" + + def test_class(self): + node = ast.ClassDef(name="X", bases=[], keywords=[], body=[ast.Pass()], decorator_list=[]) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "class X:\n pass") + + def test_class_with_type_params(self): + node = ast.ClassDef(name="X", bases=[], keywords=[], body=[ast.Pass()], decorator_list=[], + type_params=[ast.TypeVar("T")]) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "class X[T]:\n pass") + + def test_function(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f():\n pass") + + def test_function_with_type_params(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T")], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T]():\n pass") + + def test_function_with_type_params_and_bound(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T", bound=ast.Name("int"))], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") + + def test_async_function(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f():\n pass") + + def test_async_function_with_type_params(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T")], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T]():\n pass") + class DirectoryTestCase(ASTTestCase): """Test roundtrip behaviour on all files in Lib and Lib/test.""" @@ -633,7 +705,7 @@ class DirectoryTestCase(ASTTestCase): test_directories = (lib_dir, lib_dir / "test") run_always_files = {"test_grammar.py", "test_syntax.py", "test_compile.py", "test_ast.py", "test_asdl_parser.py", "test_fstring.py", - "test_patma.py"} + "test_patma.py", "test_type_alias.py", "test_type_params.py"} _files_to_test = None diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index a178e942ecda0f..9cec1e87fd3c2d 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -706,20 +706,23 @@ def test_uuid_weakref(self): self.assertIs(strong, weak()) @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-n", "@dns"]) - def test_cli_namespace_required_for_uuid3(self): + @mock.patch('sys.stderr', new_callable=io.StringIO) + def test_cli_namespace_required_for_uuid3(self, mock_err): with self.assertRaises(SystemExit) as cm: self.uuid.main() # Check that exception code is the same as argparse.ArgumentParser.error self.assertEqual(cm.exception.code, 2) + self.assertIn("error: Incorrect number of arguments", mock_err.getvalue()) @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-N", "python.org"]) - def test_cli_name_required_for_uuid3(self): + @mock.patch('sys.stderr', new_callable=io.StringIO) + def test_cli_name_required_for_uuid3(self, mock_err): with self.assertRaises(SystemExit) as cm: self.uuid.main() - # Check that exception code is the same as argparse.ArgumentParser.error self.assertEqual(cm.exception.code, 2) + self.assertIn("error: Incorrect number of arguments", mock_err.getvalue()) @mock.patch.object(sys, "argv", [""]) def test_cli_uuid4_outputted_with_no_args(self): diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 9e680c847dab7b..83237f5fe0d1b3 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -387,9 +387,13 @@ def test_catchwarnings_with_simplefilter_error(self): with self.module.catch_warnings( module=self.module, action="error", category=FutureWarning ): - self.module.warn("Other types of warnings are not errors") - self.assertRaises(FutureWarning, - self.module.warn, FutureWarning("msg")) + with support.captured_stderr() as stderr: + error_msg = "Other types of warnings are not errors" + self.module.warn(error_msg) + self.assertRaises(FutureWarning, + self.module.warn, FutureWarning("msg")) + stderr = stderr.getvalue() + self.assertIn(error_msg, stderr) class CFilterTests(FilterTests, unittest.TestCase): module = c_warnings diff --git a/Lib/test/test_zipfile/_path/__init__.py b/Lib/test/test_zipfile/_path/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_zipfile/_functools.py b/Lib/test/test_zipfile/_path/_functools.py similarity index 100% rename from Lib/test/test_zipfile/_functools.py rename to Lib/test/test_zipfile/_path/_functools.py diff --git a/Lib/test/test_zipfile/_itertools.py b/Lib/test/test_zipfile/_path/_itertools.py similarity index 100% rename from Lib/test/test_zipfile/_itertools.py rename to Lib/test/test_zipfile/_path/_itertools.py diff --git a/Lib/test/test_zipfile/_support.py b/Lib/test/test_zipfile/_path/_support.py similarity index 100% rename from Lib/test/test_zipfile/_support.py rename to Lib/test/test_zipfile/_path/_support.py diff --git a/Lib/test/test_zipfile/_test_params.py b/Lib/test/test_zipfile/_path/_test_params.py similarity index 100% rename from Lib/test/test_zipfile/_test_params.py rename to Lib/test/test_zipfile/_path/_test_params.py diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py new file mode 100644 index 00000000000000..7050937738af18 --- /dev/null +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -0,0 +1,103 @@ +import io +import itertools +import math +import re +import string +import unittest +import zipfile + +from ._functools import compose +from ._itertools import consume + +from ._support import import_or_skip + + +big_o = import_or_skip('big_o') +pytest = import_or_skip('pytest') + + +class TestComplexity(unittest.TestCase): + @pytest.mark.flaky + def test_implied_dirs_performance(self): + best, others = big_o.big_o( + compose(consume, zipfile.CompleteDirs._implied_dirs), + lambda size: [ + '/'.join(string.ascii_lowercase + str(n)) for n in range(size) + ], + max_n=1000, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + def make_zip_path(self, depth=1, width=1) -> zipfile.Path: + """ + Construct a Path with width files at every level of depth. + """ + zf = zipfile.ZipFile(io.BytesIO(), mode='w') + pairs = itertools.product(self.make_deep_paths(depth), self.make_names(width)) + for path, name in pairs: + zf.writestr(f"{path}{name}.txt", b'') + zf.filename = "big un.zip" + return zipfile.Path(zf) + + @classmethod + def make_names(cls, width, letters=string.ascii_lowercase): + """ + >>> list(TestComplexity.make_names(2)) + ['a', 'b'] + >>> list(TestComplexity.make_names(30)) + ['aa', 'ab', ..., 'bd'] + """ + # determine how many products are needed to produce width + n_products = math.ceil(math.log(width, len(letters))) + inputs = (letters,) * n_products + combinations = itertools.product(*inputs) + names = map(''.join, combinations) + return itertools.islice(names, width) + + @classmethod + def make_deep_paths(cls, depth): + return map(cls.make_deep_path, range(depth)) + + @classmethod + def make_deep_path(cls, depth): + return ''.join(('d/',) * depth) + + def test_baseline_regex_complexity(self): + best, others = big_o.big_o( + lambda path: re.fullmatch(r'[^/]*\\.txt', path), + self.make_deep_path, + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Constant + + @pytest.mark.flaky + def test_glob_depth(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + self.make_zip_path, + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Quadratic + + @pytest.mark.flaky + def test_glob_width(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + lambda size: self.make_zip_path(width=size), + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + @pytest.mark.flaky + def test_glob_width_and_depth(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + lambda size: self.make_zip_path(depth=size, width=size), + max_n=10, + min_n=1, + ) + assert best <= big_o.complexities.Linear diff --git a/Lib/test/test_zipfile/test_path.py b/Lib/test/test_zipfile/_path/test_path.py similarity index 89% rename from Lib/test/test_zipfile/test_path.py rename to Lib/test/test_zipfile/_path/test_path.py index aff91e53995875..c66cb3cba69ebd 100644 --- a/Lib/test/test_zipfile/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -41,9 +41,13 @@ def build_alpharep_fixture(): │ ├── d │ │ └── e.txt │ └── f.txt - └── g - └── h - └── i.txt + ├── g + │ └── h + │ └── i.txt + └── j + ├── k.bin + ├── l.baz + └── m.bar This fixture has the following key characteristics: @@ -51,6 +55,7 @@ def build_alpharep_fixture(): - a file two levels deep (b/d/e) - multiple files in a directory (b/c, b/f) - a directory containing only a directory (g/h) + - a directory with files of different extensions (j/klm) "alpha" because it uses alphabet "rep" because it's a representative example @@ -62,6 +67,9 @@ def build_alpharep_fixture(): zf.writestr("b/d/e.txt", b"content of e") zf.writestr("b/f.txt", b"content of f") zf.writestr("g/h/i.txt", b"content of i") + zf.writestr("j/k.bin", b"content of k") + zf.writestr("j/l.baz", b"content of l") + zf.writestr("j/m.bar", b"content of m") zf.filename = "alpharep.zip" return zf @@ -92,7 +100,7 @@ def zipfile_ondisk(self, alpharep): def test_iterdir_and_types(self, alpharep): root = zipfile.Path(alpharep) assert root.is_dir() - a, b, g = root.iterdir() + a, b, g, j = root.iterdir() assert a.is_file() assert b.is_dir() assert g.is_dir() @@ -112,7 +120,7 @@ def test_is_file_missing(self, alpharep): @pass_alpharep def test_iterdir_on_file(self, alpharep): root = zipfile.Path(alpharep) - a, b, g = root.iterdir() + a, b, g, j = root.iterdir() with self.assertRaises(ValueError): a.iterdir() @@ -127,7 +135,7 @@ def test_subdir_is_dir(self, alpharep): @pass_alpharep def test_open(self, alpharep): root = zipfile.Path(alpharep) - a, b, g = root.iterdir() + a, b, g, j = root.iterdir() with a.open(encoding="utf-8") as strm: data = strm.read() self.assertEqual(data, "content of a") @@ -229,7 +237,7 @@ def test_open_missing_directory(self): @pass_alpharep def test_read(self, alpharep): root = zipfile.Path(alpharep) - a, b, g = root.iterdir() + a, b, g, j = root.iterdir() assert a.read_text(encoding="utf-8") == "content of a" # Also check positional encoding arg (gh-101144). assert a.read_text("utf-8") == "content of a" @@ -295,7 +303,7 @@ def test_mutability(self, alpharep): reflect that change. """ root = zipfile.Path(alpharep) - a, b, g = root.iterdir() + a, b, g, j = root.iterdir() alpharep.writestr('foo.txt', 'foo') alpharep.writestr('bar/baz.txt', 'baz') assert any(child.name == 'foo.txt' for child in root.iterdir()) @@ -394,6 +402,13 @@ def test_suffixes(self, alpharep): e = root / '.hgrc' assert e.suffixes == [] + @pass_alpharep + def test_suffix_no_filename(self, alpharep): + alpharep.filename = None + root = zipfile.Path(alpharep) + assert root.joinpath('example').suffix == "" + assert root.joinpath('example').suffixes == [] + @pass_alpharep def test_stem(self, alpharep): """ @@ -411,6 +426,8 @@ def test_stem(self, alpharep): d = root / "d" assert d.stem == "d" + assert (root / ".gitignore").stem == ".gitignore" + @pass_alpharep def test_root_parent(self, alpharep): root = zipfile.Path(alpharep) @@ -442,12 +459,49 @@ def test_match_and_glob(self, alpharep): assert not root.match("*.txt") assert list(root.glob("b/c.*")) == [zipfile.Path(alpharep, "b/c.txt")] + assert list(root.glob("b/*.txt")) == [ + zipfile.Path(alpharep, "b/c.txt"), + zipfile.Path(alpharep, "b/f.txt"), + ] + @pass_alpharep + def test_glob_recursive(self, alpharep): + root = zipfile.Path(alpharep) files = root.glob("**/*.txt") assert all(each.match("*.txt") for each in files) assert list(root.glob("**/*.txt")) == list(root.rglob("*.txt")) + @pass_alpharep + def test_glob_subdirs(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("*/i.txt")) == [] + assert list(root.rglob("*/i.txt")) == [zipfile.Path(alpharep, "g/h/i.txt")] + + @pass_alpharep + def test_glob_does_not_overmatch_dot(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("*.xt")) == [] + + @pass_alpharep + def test_glob_single_char(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("a?txt")) == [zipfile.Path(alpharep, "a.txt")] + assert list(root.glob("a[.]txt")) == [zipfile.Path(alpharep, "a.txt")] + assert list(root.glob("a[?]txt")) == [] + + @pass_alpharep + def test_glob_chars(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("j/?.b[ai][nz]")) == [ + zipfile.Path(alpharep, "j/k.bin"), + zipfile.Path(alpharep, "j/l.baz"), + ] + def test_glob_empty(self): root = zipfile.Path(zipfile.ZipFile(io.BytesIO(), 'w')) with self.assertRaises(ValueError): diff --git a/Lib/test/test_zipfile/_path/write-alpharep.py b/Lib/test/test_zipfile/_path/write-alpharep.py new file mode 100644 index 00000000000000..48c09b537179fd --- /dev/null +++ b/Lib/test/test_zipfile/_path/write-alpharep.py @@ -0,0 +1,4 @@ +from . import test_path + + +__name__ == '__main__' and test_path.build_alpharep_fixture().extractall('alpharep') diff --git a/Lib/test/test_zipfile/test_complexity.py b/Lib/test/test_zipfile/test_complexity.py deleted file mode 100644 index 3432dc39e56c4e..00000000000000 --- a/Lib/test/test_zipfile/test_complexity.py +++ /dev/null @@ -1,24 +0,0 @@ -import unittest -import string -import zipfile - -from ._functools import compose -from ._itertools import consume - -from ._support import import_or_skip - - -big_o = import_or_skip('big_o') - - -class TestComplexity(unittest.TestCase): - def test_implied_dirs_performance(self): - best, others = big_o.big_o( - compose(consume, zipfile.CompleteDirs._implied_dirs), - lambda size: [ - '/'.join(string.ascii_lowercase + str(n)) for n in range(size) - ], - max_n=1000, - min_n=1, - ) - assert best <= big_o.complexities.Linear diff --git a/Lib/typing.py b/Lib/typing.py index 0b1d9689a6b7a8..9e2adbe2214a8a 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1822,14 +1822,17 @@ def __init__(cls, *args, **kwargs): def __subclasscheck__(cls, other): if cls is Protocol: return type.__subclasscheck__(cls, other) - if not isinstance(other, type): - # Same error message as for issubclass(1, int). - raise TypeError('issubclass() arg 1 must be a class') if ( getattr(cls, '_is_protocol', False) and not _allow_reckless_class_checks() ): - if not cls.__callable_proto_members_only__: + if not isinstance(other, type): + # Same error message as for issubclass(1, int). + raise TypeError('issubclass() arg 1 must be a class') + if ( + not cls.__callable_proto_members_only__ + and cls.__dict__.get("__subclasshook__") is _proto_hook + ): raise TypeError( "Protocols with non-method members don't support issubclass()" ) @@ -1873,6 +1876,30 @@ def __instancecheck__(cls, instance): return False +@classmethod +def _proto_hook(cls, other): + if not cls.__dict__.get('_is_protocol', False): + return NotImplemented + + for attr in cls.__protocol_attrs__: + for base in other.__mro__: + # Check if the members appears in the class dictionary... + if attr in base.__dict__: + if base.__dict__[attr] is None: + return NotImplemented + break + + # ...or in annotations, if it is a sub-protocol. + annotations = getattr(base, '__annotations__', {}) + if (isinstance(annotations, collections.abc.Mapping) and + attr in annotations and + issubclass(other, Generic) and getattr(other, '_is_protocol', False)): + break + else: + return NotImplemented + return True + + class Protocol(Generic, metaclass=_ProtocolMeta): """Base class for protocol classes. @@ -1918,37 +1945,11 @@ def __init_subclass__(cls, *args, **kwargs): cls._is_protocol = any(b is Protocol for b in cls.__bases__) # Set (or override) the protocol subclass hook. - def _proto_hook(other): - if not cls.__dict__.get('_is_protocol', False): - return NotImplemented - - for attr in cls.__protocol_attrs__: - for base in other.__mro__: - # Check if the members appears in the class dictionary... - if attr in base.__dict__: - if base.__dict__[attr] is None: - return NotImplemented - break - - # ...or in annotations, if it is a sub-protocol. - annotations = getattr(base, '__annotations__', {}) - if (isinstance(annotations, collections.abc.Mapping) and - attr in annotations and - issubclass(other, Generic) and getattr(other, '_is_protocol', False)): - break - else: - return NotImplemented - return True - if '__subclasshook__' not in cls.__dict__: cls.__subclasshook__ = _proto_hook - # We have nothing more to do for non-protocols... - if not cls._is_protocol: - return - - # ... otherwise prohibit instantiation. - if cls.__init__ is Protocol.__init__: + # Prohibit instantiation for protocol classes + if cls._is_protocol and cls.__init__ is Protocol.__init__: cls.__init__ = _no_init_or_replace_init diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index b73b34428764f7..c129b0d7971d71 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -52,18 +52,18 @@ uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', - 'prospero', 'rtsp', 'rtspu', 'sftp', + 'prospero', 'rtsp', 'rtsps', 'rtspu', 'sftp', 'svn', 'svn+ssh', 'ws', 'wss'] uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet', 'imap', 'wais', 'file', 'mms', 'https', 'shttp', - 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', + 'snews', 'prospero', 'rtsp', 'rtsps', 'rtspu', 'rsync', 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', 'ws', 'wss', 'itms-services'] uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', - 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', - 'mms', 'sftp', 'tel'] + 'https', 'shttp', 'rtsp', 'rtsps', 'rtspu', 'sip', + 'sips', 'mms', 'sftp', 'tel'] # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) @@ -72,7 +72,7 @@ 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms', - 'gopher', 'rtsp', 'rtspu', 'sip', 'sips'] + 'gopher', 'rtsp', 'rtsps', 'rtspu', 'sip', 'sips'] uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news', 'nntp', 'wais', 'https', 'shttp', 'snews', diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index b86d131f030d80..e0170afdf4da7d 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -391,15 +391,6 @@ def open(self, url, new=0, autoraise=True): return (p.poll() is None) - def open(self, url, new=0, autoraise=True): - sys.audit("webbrowser.open", url) - if new: - ok = self._remote("LOADNEW " + url) - else: - ok = self._remote("LOAD " + url) - return ok - - class Edge(UnixBrowser): "Launcher class for Microsoft Edge browser." diff --git a/Lib/zipfile/_path.py b/Lib/zipfile/_path/__init__.py similarity index 93% rename from Lib/zipfile/_path.py rename to Lib/zipfile/_path/__init__.py index fd49a3ea91db59..78c413563bb2b1 100644 --- a/Lib/zipfile/_path.py +++ b/Lib/zipfile/_path/__init__.py @@ -5,7 +5,8 @@ import contextlib import pathlib import re -import fnmatch + +from .glob import translate __all__ = ['Path'] @@ -296,21 +297,24 @@ def open(self, mode='r', *args, pwd=None, **kwargs): encoding, args, kwargs = _extract_text_encoding(*args, **kwargs) return io.TextIOWrapper(stream, encoding, *args, **kwargs) + def _base(self): + return pathlib.PurePosixPath(self.at or self.root.filename) + @property def name(self): - return pathlib.Path(self.at).name or self.filename.name + return self._base().name @property def suffix(self): - return pathlib.Path(self.at).suffix or self.filename.suffix + return self._base().suffix @property def suffixes(self): - return pathlib.Path(self.at).suffixes or self.filename.suffixes + return self._base().suffixes @property def stem(self): - return pathlib.Path(self.at).stem or self.filename.stem + return self._base().stem @property def filename(self): @@ -347,7 +351,7 @@ def iterdir(self): return filter(self._is_child, subs) def match(self, path_pattern): - return pathlib.Path(self.at).match(path_pattern) + return pathlib.PurePosixPath(self.at).match(path_pattern) def is_symlink(self): """ @@ -355,22 +359,13 @@ def is_symlink(self): """ return False - def _descendants(self): - for child in self.iterdir(): - yield child - if child.is_dir(): - yield from child._descendants() - def glob(self, pattern): if not pattern: raise ValueError(f"Unacceptable pattern: {pattern!r}") - matches = re.compile(fnmatch.translate(pattern)).fullmatch - return ( - child - for child in self._descendants() - if matches(str(child.relative_to(self))) - ) + prefix = re.escape(self.at) + matches = re.compile(prefix + translate(pattern)).fullmatch + return map(self._next, filter(matches, self.root.namelist())) def rglob(self, pattern): return self.glob(f'**/{pattern}') diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py new file mode 100644 index 00000000000000..4a2e665e27078a --- /dev/null +++ b/Lib/zipfile/_path/glob.py @@ -0,0 +1,40 @@ +import re + + +def translate(pattern): + r""" + Given a glob pattern, produce a regex that matches it. + + >>> translate('*.txt') + '[^/]*\\.txt' + >>> translate('a?txt') + 'a.txt' + >>> translate('**/*') + '.*/[^/]*' + """ + return ''.join(map(replace, separate(pattern))) + + +def separate(pattern): + """ + Separate out character sets to avoid translating their contents. + + >>> [m.group(0) for m in separate('*.txt')] + ['*.txt'] + >>> [m.group(0) for m in separate('a[?]txt')] + ['a', '[?]', 'txt'] + """ + return re.finditer(r'([^\[]+)|(?P[\[].*?[\]])|([\[][^\]]*$)', pattern) + + +def replace(match): + """ + Perform the replacements for a match from :func:`separate`. + """ + + return match.group('set') or ( + re.escape(match.group(0)) + .replace('\\*\\*', r'.*') + .replace('\\*', r'[^/]*') + .replace('\\?', r'.') + ) diff --git a/Makefile.pre.in b/Makefile.pre.in index e2adc3cb49f2a9..8dacb570ccafab 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2014,7 +2014,11 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ fi .PHONY: bininstall -bininstall: altbininstall +# We depend on commoninstall here to make sure the installation is already usable +# before we possibly overwrite the global 'python3' symlink to avoid causing +# problems for anything else trying to run 'python3' while we install, particularly +# if we're installing in parallel with -j. +bininstall: commoninstall altbininstall if test ! -d $(DESTDIR)$(LIBPC); then \ echo "Creating directory $(LIBPC)"; \ $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(LIBPC); \ @@ -2106,7 +2110,7 @@ LIBSUBDIRS= asyncio \ wsgiref \ $(XMLLIBSUBDIRS) \ xmlrpc \ - zipfile \ + zipfile zipfile/_path \ zoneinfo \ __phello__ TESTSUBDIRS= idlelib/idle_test \ @@ -2216,6 +2220,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_warnings \ test/test_warnings/data \ test/test_zipfile \ + test/test_zipfile/_path \ test/test_zoneinfo \ test/test_zoneinfo/data \ test/tracedmodules \ diff --git a/Misc/ACKS b/Misc/ACKS index be8755637ffa3c..645ad5b700baaa 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -226,6 +226,7 @@ Erik Bray Brian Brazil Demian Brecht Dave Brennan +Christopher Richard James Brett Tom Bridgman Anthony Briggs Keith Briggs @@ -615,6 +616,7 @@ Marius Gedminas Jan-Philip Gehrcke Thomas Gellekum Gabriel Genellina +Andrew Geng Philip Georgi Christos Georgiou Elazar (אלעזר) Gershuni @@ -1946,6 +1948,7 @@ Colin Watson David Watson Aaron Watters Alex Waygood +James Webber Russel Webber Henrik Weber Leon Weber diff --git a/Misc/NEWS.d/3.12.0b3.rst b/Misc/NEWS.d/3.12.0b3.rst new file mode 100644 index 00000000000000..ffd880f8688679 --- /dev/null +++ b/Misc/NEWS.d/3.12.0b3.rst @@ -0,0 +1,411 @@ +.. date: 2023-06-15-22-11-43 +.. gh-issue: 105840 +.. nonce: Fum_g_ +.. release date: 2023-06-19 +.. section: Core and Builtins + +Fix possible crashes when specializing function calls with too many +``__defaults__``. + +.. + +.. date: 2023-06-15-15-54-47 +.. gh-issue: 105831 +.. nonce: -MC9Zs +.. section: Core and Builtins + +Fix an f-string bug, where using a debug expression (the ``=`` sign) that +appears in the last line of a file results to the debug buffer that holds +the expression text being one character too small. + +.. + +.. date: 2023-06-14-22-52-06 +.. gh-issue: 105800 +.. nonce: hdpPzZ +.. section: Core and Builtins + +Correctly issue :exc:`SyntaxWarning` in f-strings if invalid sequences are +used. Patch by Pablo Galindo + +.. + +.. date: 2023-06-10-21-38-49 +.. gh-issue: 105587 +.. nonce: rL3rzv +.. section: Core and Builtins + +The runtime can't guarantee that immortal objects will not be mutated by +Extensions. Thus, this modifies _PyStaticObject_CheckRefcnt to warn instead +of asserting. + +.. + +.. date: 2023-06-09-15-25-12 +.. gh-issue: 105564 +.. nonce: sFdUu4 +.. section: Core and Builtins + +Don't include artificil newlines in the ``line`` attribute of tokens in the +APIs of the :mod:`tokenize` module. Patch by Pablo Galindo + +.. + +.. date: 2023-06-09-12-59-18 +.. gh-issue: 105549 +.. nonce: PYfTNp +.. section: Core and Builtins + +Tokenize separately ``NUMBER`` and ``NAME`` tokens that are not ambiguous. +Patch by Pablo Galindo. + +.. + +.. date: 2023-06-09-11-19-51 +.. gh-issue: 105588 +.. nonce: Y5ovpY +.. section: Core and Builtins + +Fix an issue that could result in crashes when compiling malformed +:mod:`ast` nodes. + +.. + +.. date: 2023-06-08-10-10-07 +.. gh-issue: 105375 +.. nonce: 35VGDd +.. section: Core and Builtins + +Fix bugs in the :mod:`builtins` module where exceptions could end up being +overwritten. + +.. + +.. date: 2023-06-08-09-54-37 +.. gh-issue: 105375 +.. nonce: kqKT3E +.. section: Core and Builtins + +Fix bug in the compiler where an exception could end up being overwritten. + +.. + +.. date: 2023-06-08-09-25-52 +.. gh-issue: 105375 +.. nonce: ocB7fT +.. section: Core and Builtins + +Improve error handling in :c:func:`PyUnicode_BuildEncodingMap` where an +exception could end up being overwritten. + +.. + +.. date: 2023-06-07-12-20-59 +.. gh-issue: 105435 +.. nonce: 6VllI0 +.. section: Core and Builtins + +Fix spurious newline character if file ends on a comment without a newline. +Patch by Pablo Galindo + +.. + +.. date: 2023-06-06-17-10-42 +.. gh-issue: 105390 +.. nonce: DvqI-e +.. section: Core and Builtins + +Correctly raise :exc:`tokenize.TokenError` exceptions instead of +:exc:`SyntaxError` for tokenize errors such as incomplete input. Patch by +Pablo Galindo + +.. + +.. date: 2023-06-02-15-15-41 +.. gh-issue: 104812 +.. nonce: dfZiG5 +.. section: Core and Builtins + +The "pending call" machinery now works for all interpreters, not just the +main interpreter, and runs in all threads, not just the main thread. Some +calls are still only done in the main thread, ergo in the main interpreter. +This change does not affect signal handling nor the existing public C-API +(``Py_AddPendingCall()``), which both still only target the main thread. The +new functionality is meant strictly for internal use for now, since +consequences of its use are not well understood yet outside some very +restricted cases. This change brings the capability in line with the +intention when the state was made per-interpreter several years ago. + +.. + +.. date: 2023-06-19-11-31-55 +.. gh-issue: 105808 +.. nonce: NL-quu +.. section: Library + +Fix a regression introduced in GH-101251 for 3.12, causing +:meth:`gzip.GzipFile.flush` to not flush the compressor (nor pass along the +``zip_mode`` argument). + +.. + +.. date: 2023-06-15-18-11-47 +.. gh-issue: 104799 +.. nonce: BcLzbP +.. section: Library + +Enable :func:`ast.unparse` to unparse function and class definitions created +without the new ``type_params`` field from :pep:`695`. Patch by Jelle +Zijlstra. + +.. + +.. date: 2023-06-14-10-27-34 +.. gh-issue: 105745 +.. nonce: l1ttOQ +.. section: Library + +Fix ``webbrowser.Konqueror.open`` method. + +.. + +.. date: 2023-06-11-22-46-06 +.. gh-issue: 105375 +.. nonce: YkhSNt +.. section: Library + +Fix a bug in :c:func:`!_Unpickler_SetInputStream` where an exception could +end up being overwritten in case of failure. + +.. + +.. date: 2023-06-09-23-46-23 +.. gh-issue: 105375 +.. nonce: 9KaioS +.. section: Library + +Fix bugs in :mod:`sys` where exceptions could end up being overwritten +because of deferred error handling. + +.. + +.. date: 2023-06-09-23-00-13 +.. gh-issue: 105605 +.. nonce: YuwqxY +.. section: Library + +Harden :mod:`pyexpat` error handling during module initialisation to prevent +exceptions from possibly being overwritten, and objects from being +dereferenced twice. + +.. + +.. date: 2023-06-09-22-52-45 +.. gh-issue: 105375 +.. nonce: 6igkhn +.. section: Library + +Fix bug in :mod:`decimal` where an exception could end up being overwritten. + +.. + +.. date: 2023-06-09-22-45-26 +.. gh-issue: 105375 +.. nonce: 9rp6tG +.. section: Library + +Fix bugs in :mod:`!_datetime` where exceptions could be overwritten in case +of module initialisation failure. + +.. + +.. date: 2023-06-09-22-16-46 +.. gh-issue: 105375 +.. nonce: EgVJOP +.. section: Library + +Fix bugs in :mod:`!_ssl` initialisation which could lead to leaked +references and overwritten exceptions. + +.. + +.. date: 2023-06-09-21-46-52 +.. gh-issue: 105375 +.. nonce: yrJelV +.. section: Library + +Fix a bug in :class:`array.array` where an exception could end up being +overwritten. + +.. + +.. date: 2023-06-09-21-40-45 +.. gh-issue: 105375 +.. nonce: _sZilh +.. section: Library + +Fix bugs in :mod:`_ctypes` where exceptions could end up being overwritten. + +.. + +.. date: 2023-06-09-21-30-59 +.. gh-issue: 105375 +.. nonce: eewafp +.. section: Library + +Fix a bug in the :mod:`posix` module where an exception could be +overwritten. + +.. + +.. date: 2023-06-09-21-25-14 +.. gh-issue: 105375 +.. nonce: 95g1eI +.. section: Library + +Fix bugs in :mod:`!_elementtree` where exceptions could be overwritten. + +.. + +.. date: 2023-06-09-21-11-28 +.. gh-issue: 105375 +.. nonce: 4Mxn7t +.. section: Library + +Fix bugs in :mod:`zoneinfo` where exceptions could be overwritten. + +.. + +.. date: 2023-06-09-21-04-39 +.. gh-issue: 105375 +.. nonce: bTcqS9 +.. section: Library + +Fix bugs in :mod:`pickle` where exceptions could be overwritten. + +.. + +.. date: 2023-06-08-08-58-36 +.. gh-issue: 105375 +.. nonce: bTcqS9 +.. section: Library + +Fix bugs in :mod:`pickle` where exceptions could be overwritten. + +.. + +.. date: 2023-06-07-00-09-52 +.. gh-issue: 105375 +.. nonce: Y_9D4n +.. section: Library + +Fix a bug in :mod:`sqlite3` where an exception could be overwritten in the +:meth:`collation ` callback. + +.. + +.. date: 2023-06-06-11-50-33 +.. gh-issue: 105332 +.. nonce: tmpgRA +.. section: Library + +Revert pickling method from by-name back to by-value. + +.. + +.. date: 2023-06-02-14-23-41 +.. gh-issue: 104310 +.. nonce: UamCOB +.. section: Library + +In the beta 1 release we added a utility function for extension module +authors, to use when testing their module for support in multiple +interpreters or under a per-interpreter GIL. The name of that function has +changed from ``allowing_all_extensions`` to +``_incompatible_extension_module_restrictions``. The default for the +"disable_check" argument has change from ``True`` to ``False``, to better +match the new function name. + +.. + +.. date: 2023-05-26-21-24-06 +.. gh-issue: 104996 +.. nonce: aaW78g +.. section: Library + +Improve performance of :class:`pathlib.PurePath` initialisation by deferring +joining of paths when multiple arguments are given. + +.. + +.. date: 2023-03-12-01-17-15 +.. gh-issue: 102541 +.. nonce: LK1adc +.. section: Library + +Hide traceback in :func:`help` prompt, when import failed. + +.. + +.. date: 2023-05-29-14-49-46 +.. gh-issue: 105084 +.. nonce: lvVvoj +.. section: Tests + +When the Python build is configured ``--with-wheel-pkg-dir``, tests +requiring the ``setuptools`` and ``wheel`` wheels will search for the wheels +in ``WHEEL_PKG_DIR``. + +.. + +.. date: 2023-06-08-11-30-17 +.. gh-issue: 105436 +.. nonce: 1qlDxw +.. section: Windows + +Ensure that an empty environment block is terminated by two null characters, +as is required by Windows. + +.. + +.. date: 2023-06-09-23-34-25 +.. gh-issue: 105375 +.. nonce: n7amiF +.. section: C API + +Fix a bug in :c:func:`PyErr_WarnExplicit` where an exception could end up +being overwritten if the API failed internally. + +.. + +.. date: 2023-06-09-19-16-57 +.. gh-issue: 105603 +.. nonce: -z6G22 +.. section: C API + +We've renamed the new (in 3.12) ``PyInterpreterConfig.own_gil`` to +``PyInterpreterConfig.gil`` and changed the meaning of the value from "bool" +to an integer with supported values of ``PyInterpreterConfig_DEFAULT_GIL``, +``PyInterpreterConfig_SHARED_GIL``, and ``PyInterpreterConfig_OWN_GIL``. The +default is "shared". + +.. + +.. date: 2023-06-09-12-35-55 +.. gh-issue: 105387 +.. nonce: wM_oL- +.. section: C API + +In the limited C API version 3.12, :c:func:`Py_INCREF` and +:c:func:`Py_DECREF` functions are now implemented as opaque function calls +to hide implementation details. Patch by Victor Stinner. + +.. + +.. date: 2023-06-06-14-14-41 +.. gh-issue: 103968 +.. nonce: BTO6II +.. section: C API + +:c:func:`PyType_FromMetaclass` now allows metaclasses with ``tp_new`` set to +``NULL``. diff --git a/Misc/NEWS.d/3.12.0b4.rst b/Misc/NEWS.d/3.12.0b4.rst new file mode 100644 index 00000000000000..5212f359ec5063 --- /dev/null +++ b/Misc/NEWS.d/3.12.0b4.rst @@ -0,0 +1,318 @@ +.. date: 2023-06-13-20-52-24 +.. gh-issue: 102988 +.. nonce: Kei7Vf +.. release date: 2023-07-11 +.. section: Security + +CVE-2023-27043: Prevent :func:`email.utils.parseaddr` and +:func:`email.utils.getaddresses` from returning the realname portion of an +invalid RFC2822 email header in the email address portion of the 2-tuple +returned after being parsed by :class:`email._parseaddr.AddressList`. + +.. + +.. date: 2023-07-04-09-51-45 +.. gh-issue: 106396 +.. nonce: DmYp7x +.. section: Core and Builtins + +When the format specification of an f-string expression is empty, the parser +now generates an empty :class:`ast.JoinedStr` node for it instead of an +one-element :class:`ast.JoinedStr` with an empty string +:class:`ast.Constant`. + +.. + +.. date: 2023-06-29-09-46-41 +.. gh-issue: 106145 +.. nonce: QC6-Kq +.. section: Core and Builtins + +Make ``end_lineno`` and ``end_col_offset`` required on ``type_param`` ast +nodes. + +.. + +.. date: 2023-06-22-19-16-24 +.. gh-issue: 105979 +.. nonce: TDP2CU +.. section: Core and Builtins + +Fix crash in :func:`!_imp.get_frozen_object` due to improper exception +handling. + +.. + +.. date: 2023-06-22-14-19-17 +.. gh-issue: 98931 +.. nonce: PPgvSF +.. section: Core and Builtins + +Ensure custom :exc:`SyntaxError` error messages are raised for invalid +imports with multiple targets. Patch by Pablo Galindo + +.. + +.. date: 2023-06-19-11-04-01 +.. gh-issue: 105908 +.. nonce: 7oanny +.. section: Core and Builtins + +Fixed bug where :gh:`99111` breaks future import ``barry_as_FLUFL`` in the +Python REPL. + +.. + +.. date: 2023-06-12-16-38-31 +.. gh-issue: 105340 +.. nonce: _jRHXe +.. section: Core and Builtins + +Include the comprehension iteration variable in ``locals()`` inside a +module- or class-scope comprehension. + +.. + +.. date: 2023-06-08-09-10-15 +.. gh-issue: 105486 +.. nonce: dev-WS +.. section: Core and Builtins + +Change the repr of ``ParamSpec`` list of args in ``types.GenericAlias``. + +.. + +.. date: 2023-01-13-11-37-41 +.. gh-issue: 101006 +.. nonce: fuLvn2 +.. section: Core and Builtins + +Improve error handling when read :mod:`marshal` data. + +.. + +.. date: 2023-07-07-17-44-03 +.. gh-issue: 106524 +.. nonce: XkBV8h +.. section: Library + +Fix crash in :func:`!_sre.template` with templates containing invalid group +indices. + +.. + +.. date: 2023-07-07-13-47-28 +.. gh-issue: 106510 +.. nonce: 9n5BdC +.. section: Library + +Improve debug output for atomic groups in regular expressions. + +.. + +.. date: 2023-07-07-03-05-58 +.. gh-issue: 106503 +.. nonce: ltfeiH +.. section: Library + +Fix ref cycle in :class:`!asyncio._SelectorSocketTransport` by removing +``_write_ready`` in ``close``. + +.. + +.. date: 2023-07-05-14-34-10 +.. gh-issue: 105497 +.. nonce: HU5u89 +.. section: Library + +Fix flag mask inversion when unnamed flags exist. + +.. + +.. date: 2023-07-05-13-08-23 +.. gh-issue: 90876 +.. nonce: Qvlkfl +.. section: Library + +Prevent :mod:`multiprocessing.spawn` from failing to *import* in +environments where ``sys.executable`` is ``None``. This regressed in 3.11 +with the addition of support for path-like objects in multiprocessing. + +.. + +.. date: 2023-07-03-15-09-44 +.. gh-issue: 106292 +.. nonce: 3npldV +.. section: Library + +Check for an instance-dict cached value in the :meth:`__get__` method of +:func:`functools.cached_property`. This better matches the pre-3.12 behavior +and improves compatibility for users subclassing +:func:`functools.cached_property` and adding a :meth:`__set__` method. + +.. + +.. date: 2023-07-02-10-56-41 +.. gh-issue: 106330 +.. nonce: QSkIUH +.. section: Library + +Fix incorrect matching of empty paths in :meth:`pathlib.PurePath.match`. +This bug was introduced in Python 3.12.0 beta 1. + +.. + +.. date: 2023-07-01-16-40-54 +.. gh-issue: 102541 +.. nonce: C1ahtk +.. section: Library + +Make pydoc.doc catch bad module ImportError when output stream is not None. + +.. + +.. date: 2023-06-27-23-22-37 +.. gh-issue: 106152 +.. nonce: ya5jBT +.. section: Library + +Added PY_THROW event hook for :mod:`cProfile` for generators + +.. + +.. date: 2023-06-25-12-28-55 +.. gh-issue: 106075 +.. nonce: W7tMRb +.. section: Library + +Added `asyncio.taskgroups.__all__` to `asyncio.__all__` for export in star +imports. + +.. + +.. date: 2023-06-22-15-21-11 +.. gh-issue: 105987 +.. nonce: T7Kzrb +.. section: Library + +Fix crash due to improper reference counting in :mod:`asyncio` eager task +factory internal routines. + +.. + +.. date: 2023-06-21-19-04-27 +.. gh-issue: 105974 +.. nonce: M47n3t +.. section: Library + +Fix bug where a :class:`typing.Protocol` class that had one or more +non-callable members would raise :exc:`TypeError` when :func:`issubclass` +was called against it, even if it defined a custom ``__subclasshook__`` +method. The behaviour in Python 3.11 and lower -- which has now been +restored -- was not to raise :exc:`TypeError` in these situations if a +custom ``__subclasshook__`` method was defined. Patch by Alex Waygood. + +.. + +.. date: 2023-06-20-23-18-45 +.. gh-issue: 96145 +.. nonce: o5dTRM +.. section: Library + +Reverted addition of ``json.AttrDict``. + +.. + +.. date: 2023-06-08-17-49-46 +.. gh-issue: 105497 +.. nonce: K6Q8nU +.. section: Library + +Fix flag inversion when alias/mask members exist. + +.. + +.. date: 2023-06-05-14-43-56 +.. gh-issue: 104554 +.. nonce: pwfKIo +.. section: Library + +Add RTSPS scheme support in urllib.parse + +.. + +.. date: 2022-07-12-18-45-13 +.. gh-issue: 94777 +.. nonce: mOybx7 +.. section: Library + +Fix hanging :mod:`multiprocessing` ``ProcessPoolExecutor`` when a child +process crashes while data is being written in the call queue. + +.. + +.. date: 2023-06-30-19-28-59 +.. gh-issue: 106232 +.. nonce: hQ4-tz +.. section: Documentation + +Make timeit doc command lines compatible with Windows by using double quotes +for arguments. This works on linux and macOS also. + +.. + +.. date: 2023-06-28-02-51-08 +.. gh-issue: 101634 +.. nonce: Rayczr +.. section: Tests + +When running the Python test suite with ``-jN`` option, if a worker stdout +cannot be decoded from the locale encoding report a failed testn so the +exitcode is non-zero. Patch by Victor Stinner. + +.. + +.. date: 2023-06-26-21-56-29 +.. gh-issue: 106118 +.. nonce: 0cCfhl +.. section: Build + +Fix compilation for platforms without :data:`!O_CLOEXEC`. The issue was +introduced with Python 3.12b1 in :gh:`103295`. Patch by Erlend Aasland. + +.. + +.. date: 2023-05-20-23-49-30 +.. gh-issue: 104692 +.. nonce: s5UIu5 +.. section: Build + +Include ``commoninstall`` as a prerequisite for ``bininstall`` + +This ensures that ``commoninstall`` is completed before ``bininstall`` is +started when parallel builds are used (``make -j install``), and so the +``python3`` symlink is only installed after all standard library modules are +installed. + +.. + +.. date: 2023-07-03-14-06-19 +.. gh-issue: 106359 +.. nonce: RfJuR0 +.. section: Tools/Demos + +Argument Clinic now explicitly forbids "kwarg splats" in function calls used +as annotations. + +.. + +.. date: 2023-06-13-14-24-55 +.. gh-issue: 105227 +.. nonce: HDL9aF +.. section: C API + +The new :c:func:`PyType_GetDict` provides the dictionary for the given type +object that is normally exposed by ``cls.__dict__``. Normally it's +sufficient to use :c:member:`~PyTypeObject.tp_dict`, but for the static +builtin types :c:member:`!tp_dict` is now always ``NULL``. +:c:func:`!PyType_GetDict()` provides the correct dict object instead. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst new file mode 100644 index 00000000000000..c28d0101cd4bad --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst @@ -0,0 +1 @@ +Prevent out-of-bounds memory access during ``mmap.find()`` calls. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-06-17-10-42.gh-issue-105390.DvqI-e.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-06-17-10-42.gh-issue-105390.DvqI-e.rst deleted file mode 100644 index de59b54d8f6053..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-06-17-10-42.gh-issue-105390.DvqI-e.rst +++ /dev/null @@ -1,3 +0,0 @@ -Correctly raise :exc:`tokenize.TokenError` exceptions instead of -:exc:`SyntaxError` for tokenize errors such as incomplete input. Patch by -Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-07-12-20-59.gh-issue-105435.6VllI0.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-07-12-20-59.gh-issue-105435.6VllI0.rst deleted file mode 100644 index 9e4d7e1851ccb5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-07-12-20-59.gh-issue-105435.6VllI0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix spurious newline character if file ends on a comment without a newline. -Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-08-09-54-37.gh-issue-105375.kqKT3E.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-08-09-54-37.gh-issue-105375.kqKT3E.rst deleted file mode 100644 index b4d3a1a5a3cedb..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-08-09-54-37.gh-issue-105375.kqKT3E.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug in the compiler where an exception could end up being overwritten. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-09-12-59-18.gh-issue-105549.PYfTNp.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-09-12-59-18.gh-issue-105549.PYfTNp.rst deleted file mode 100644 index 7cb177b9353373..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-09-12-59-18.gh-issue-105549.PYfTNp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Tokenize separately ``NUMBER`` and ``NAME`` tokens that are not ambiguous. Patch -by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-09-15-25-12.gh-issue-105564.sFdUu4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-09-15-25-12.gh-issue-105564.sFdUu4.rst deleted file mode 100644 index 9809fac49164f5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-09-15-25-12.gh-issue-105564.sFdUu4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Don't include artificil newlines in the ``line`` attribute of tokens in the -APIs of the :mod:`tokenize` module. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Library/2023-05-26-21-24-06.gh-issue-104996.aaW78g.rst b/Misc/NEWS.d/next/Library/2023-05-26-21-24-06.gh-issue-104996.aaW78g.rst deleted file mode 100644 index 8b81b681af94aa..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-26-21-24-06.gh-issue-104996.aaW78g.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :class:`pathlib.PurePath` initialisation by -deferring joining of paths when multiple arguments are given. diff --git a/Misc/NEWS.d/next/Library/2023-06-02-14-23-41.gh-issue-104310.UamCOB.rst b/Misc/NEWS.d/next/Library/2023-06-02-14-23-41.gh-issue-104310.UamCOB.rst deleted file mode 100644 index 461a3a25fe1b43..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-02-14-23-41.gh-issue-104310.UamCOB.rst +++ /dev/null @@ -1,7 +0,0 @@ -In the beta 1 release we added a utility function for extension module -authors, to use when testing their module for support in multiple -interpreters or under a per-interpreter GIL. The name of that function has -changed from ``allowing_all_extensions`` to -``_incompatible_extension_module_restrictions``. The default for the -"disable_check" argument has change from ``True`` to ``False``, to better -match the new function name. diff --git a/Misc/NEWS.d/next/Library/2023-06-06-11-50-33.gh-issue-105332.tmpgRA.rst b/Misc/NEWS.d/next/Library/2023-06-06-11-50-33.gh-issue-105332.tmpgRA.rst deleted file mode 100644 index 31b6855a6ebfad..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-06-11-50-33.gh-issue-105332.tmpgRA.rst +++ /dev/null @@ -1 +0,0 @@ -Revert pickling method from by-name back to by-value. diff --git a/Misc/NEWS.d/next/Library/2023-06-07-00-09-52.gh-issue-105375.Y_9D4n.rst b/Misc/NEWS.d/next/Library/2023-06-07-00-09-52.gh-issue-105375.Y_9D4n.rst deleted file mode 100644 index ec10d63822c203..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-07-00-09-52.gh-issue-105375.Y_9D4n.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug in :mod:`sqlite3` where an exception could be overwritten in the -:meth:`collation ` callback. diff --git a/Misc/NEWS.d/next/Library/2023-06-08-08-58-36.gh-issue-105375.bTcqS9.rst b/Misc/NEWS.d/next/Library/2023-06-08-08-58-36.gh-issue-105375.bTcqS9.rst deleted file mode 100644 index 3030477c8245b5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-08-08-58-36.gh-issue-105375.bTcqS9.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bugs in :mod:`pickle` where exceptions could be overwritten. diff --git a/Misc/NEWS.d/next/Library/2023-06-09-21-04-39.gh-issue-105375.bTcqS9.rst b/Misc/NEWS.d/next/Library/2023-06-09-21-04-39.gh-issue-105375.bTcqS9.rst deleted file mode 100644 index 3030477c8245b5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-09-21-04-39.gh-issue-105375.bTcqS9.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bugs in :mod:`pickle` where exceptions could be overwritten. diff --git a/Misc/NEWS.d/next/Library/2023-06-09-21-11-28.gh-issue-105375.4Mxn7t.rst b/Misc/NEWS.d/next/Library/2023-06-09-21-11-28.gh-issue-105375.4Mxn7t.rst deleted file mode 100644 index 4202b758d1db56..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-09-21-11-28.gh-issue-105375.4Mxn7t.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bugs in :mod:`zoneinfo` where exceptions could be overwritten. diff --git a/Misc/NEWS.d/next/Library/2023-06-09-21-25-14.gh-issue-105375.95g1eI.rst b/Misc/NEWS.d/next/Library/2023-06-09-21-25-14.gh-issue-105375.95g1eI.rst deleted file mode 100644 index 1894b2b94bb334..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-09-21-25-14.gh-issue-105375.95g1eI.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bugs in :mod:`!_elementtree` where exceptions could be overwritten. diff --git a/Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst b/Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst new file mode 100644 index 00000000000000..2a48361fa596c9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst @@ -0,0 +1,3 @@ +Change the default return value of +:meth:`http.client.HTTPConnection.get_proxy_response_headers` to be ``None`` +and not ``{}``. diff --git a/Misc/NEWS.d/next/Library/2023-07-04-07-25-30.gh-issue-106403.GmefbV.rst b/Misc/NEWS.d/next/Library/2023-07-04-07-25-30.gh-issue-106403.GmefbV.rst new file mode 100644 index 00000000000000..4fea45f16c4f8e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-04-07-25-30.gh-issue-106403.GmefbV.rst @@ -0,0 +1,4 @@ +Instances of :class:`typing.TypeVar`, :class:`typing.ParamSpec`, +:class:`typing.ParamSpecArgs`, :class:`typing.ParamSpecKwargs`, and +:class:`typing.TypeVarTuple` once again support weak references, fixing a +regression introduced in Python 3.12.0 beta 1. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2023-07-11-09-25-40.gh-issue-106530.VgXrMx.rst b/Misc/NEWS.d/next/Library/2023-07-11-09-25-40.gh-issue-106530.VgXrMx.rst new file mode 100644 index 00000000000000..09fc647cc01d21 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-11-09-25-40.gh-issue-106530.VgXrMx.rst @@ -0,0 +1,2 @@ +Revert a change to :func:`colorsys.rgb_to_hls` that caused division by zero +for certain almost-white inputs. Patch by Terry Jan Reedy. diff --git a/Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst b/Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst new file mode 100644 index 00000000000000..d9c122f1d3c723 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst @@ -0,0 +1 @@ +Add __copy__ and __deepcopy__ in :mod:`enum` diff --git a/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst b/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst new file mode 100644 index 00000000000000..bbc53d76decbc3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst @@ -0,0 +1,5 @@ +Fixed several bugs in zipfile.Path, including: in ``Path.match`, Windows +separators are no longer honored (and never were meant to be); Fixed +``name``/``suffix``/``suffixes``/``stem`` operations when no filename is +present and the Path is not at the root of the zipfile; Reworked glob for +performance and more correct matching behavior. diff --git a/Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst b/Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst new file mode 100644 index 00000000000000..ba7257e3610808 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst @@ -0,0 +1,2 @@ +Moved tests for ``zipfile.Path`` into ``Lib/test/test_zipfile/_path``. Made +``zipfile._path`` a package. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst b/Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst new file mode 100644 index 00000000000000..7e28ba6963216a --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst @@ -0,0 +1,2 @@ +``freeze`` now fetches ``CONFIG_ARGS`` from the original CPython instance +the Makefile uses to call utility scripts. Patch by Ijtaba Hussain. diff --git a/Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst b/Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst new file mode 100644 index 00000000000000..11f411be0f17c5 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst @@ -0,0 +1 @@ +Update Windows build to use OpenSSL 3.0.9 diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 7e33558dba3e32..a465090bfaaa38 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2047,20 +2047,23 @@ swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) } prev_task = Py_None; } + Py_INCREF(prev_task); if (task == Py_None) { if (_PyDict_DelItem_KnownHash(state->current_tasks, loop, hash) == -1) { - return NULL; + goto error; } } else { if (_PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash) == -1) { - return NULL; + goto error; } } - Py_INCREF(prev_task); - return prev_task; + +error: + Py_DECREF(prev_task); + return NULL; } /* ----- Task */ diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 8e694ba852c1d4..d71297f9c5c2f2 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -479,12 +479,22 @@ long Call_GetClassObject(REFCLSID rclsid, REFIID riid, LPVOID *ppv) { PyObject *py_rclsid = PyLong_FromVoidPtr((void *)rclsid); + if (py_rclsid == NULL) { + Py_DECREF(func); + PyErr_WriteUnraisable(context ? context : Py_None); + return E_FAIL; + } PyObject *py_riid = PyLong_FromVoidPtr((void *)riid); + if (py_riid == NULL) { + Py_DECREF(func); + Py_DECREF(py_rclsid); + PyErr_WriteUnraisable(context ? context : Py_None); + return E_FAIL; + } PyObject *py_ppv = PyLong_FromVoidPtr(ppv); - if (!py_rclsid || !py_riid || !py_ppv) { - Py_XDECREF(py_rclsid); - Py_XDECREF(py_riid); - Py_XDECREF(py_ppv); + if (py_ppv == NULL) { + Py_DECREF(py_rclsid); + Py_DECREF(py_riid); Py_DECREF(func); PyErr_WriteUnraisable(context ? context : Py_None); return E_FAIL; diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 19e11780ec6e19..cbc331c7e94268 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -6871,24 +6871,49 @@ _datetime_exec(PyObject *module) assert(DI100Y == days_before_year(100+1)); us_per_ms = PyLong_FromLong(1000); + if (us_per_ms == NULL) { + goto error; + } us_per_second = PyLong_FromLong(1000000); + if (us_per_second == NULL) { + goto error; + } us_per_minute = PyLong_FromLong(60000000); + if (us_per_minute == NULL) { + goto error; + } seconds_per_day = PyLong_FromLong(24 * 3600); - if (us_per_ms == NULL || us_per_second == NULL || - us_per_minute == NULL || seconds_per_day == NULL) { - return -1; + if (seconds_per_day == NULL) { + goto error; } /* The rest are too big for 32-bit ints, but even * us_per_week fits in 40 bits, so doubles should be exact. */ us_per_hour = PyLong_FromDouble(3600000000.0); + if (us_per_hour == NULL) { + goto error; + } us_per_day = PyLong_FromDouble(86400000000.0); + if (us_per_day == NULL) { + goto error; + } us_per_week = PyLong_FromDouble(604800000000.0); - if (us_per_hour == NULL || us_per_day == NULL || us_per_week == NULL) { - return -1; + if (us_per_week == NULL) { + goto error; } + return 0; + +error: + Py_XDECREF(us_per_ms); + Py_XDECREF(us_per_second); + Py_XDECREF(us_per_minute); + Py_XDECREF(us_per_hour); + Py_XDECREF(us_per_day); + Py_XDECREF(us_per_week); + Py_XDECREF(seconds_per_day); + return -1; } static struct PyModuleDef datetimemodule = { diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 0e11c879732ab6..0005081f7c5e80 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -3615,9 +3615,13 @@ dec_as_integer_ratio(PyObject *self, PyObject *args UNUSED) goto error; } Py_SETREF(numerator, _py_long_floor_divide(numerator, tmp)); + if (numerator == NULL) { + Py_DECREF(tmp); + goto error; + } Py_SETREF(denominator, _py_long_floor_divide(denominator, tmp)); Py_DECREF(tmp); - if (numerator == NULL || denominator == NULL) { + if (denominator == NULL) { goto error; } } diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 4b425f4147513e..af6d1b23d3ae91 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -383,14 +383,15 @@ py_digest_by_digestmod(PyObject *module, PyObject *digestmod, enum Py_hash_type } else { _hashlibstate *state = get_hashlib_state(module); // borrowed ref - name_obj = PyDict_GetItem(state->constructs, digestmod); + name_obj = PyDict_GetItemWithError(state->constructs, digestmod); } if (name_obj == NULL) { - _hashlibstate *state = get_hashlib_state(module); - PyErr_Clear(); - PyErr_Format( - state->unsupported_digestmod_error, - "Unsupported digestmod %R", digestmod); + if (!PyErr_Occurred()) { + _hashlibstate *state = get_hashlib_state(module); + PyErr_Format( + state->unsupported_digestmod_error, + "Unsupported digestmod %R", digestmod); + } return NULL; } diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 1c84f66ee6f579..257de4387c0ab9 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -678,6 +678,7 @@ static const struct { } callback_table[] = { {PY_MONITORING_EVENT_PY_START, "_pystart_callback"}, {PY_MONITORING_EVENT_PY_RESUME, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_THROW, "_pystart_callback"}, {PY_MONITORING_EVENT_PY_RETURN, "_pyreturn_callback"}, {PY_MONITORING_EVENT_PY_YIELD, "_pyreturn_callback"}, {PY_MONITORING_EVENT_PY_UNWIND, "_pyreturn_callback"}, diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d6a273b1044e44..d2f6d71059ea00 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1145,42 +1145,49 @@ _Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t data_len) static PicklerObject * _Pickler_New(PickleState *st) { - PicklerObject *self; - - self = PyObject_GC_New(PicklerObject, st->Pickler_Type); - if (self == NULL) + PyMemoTable *memo = PyMemoTable_New(); + if (memo == NULL) { return NULL; + } + + const Py_ssize_t max_output_len = WRITE_BUF_SIZE; + PyObject *output_buffer = PyBytes_FromStringAndSize(NULL, max_output_len); + if (output_buffer == NULL) { + goto error; + } + + PicklerObject *self = PyObject_GC_New(PicklerObject, st->Pickler_Type); + if (self == NULL) { + goto error; + } + self->memo = memo; self->pers_func = NULL; + self->pers_func_self = NULL; self->dispatch_table = NULL; - self->buffer_callback = NULL; + self->reducer_override = NULL; self->write = NULL; + self->output_buffer = output_buffer; + self->output_len = 0; + self->max_output_len = max_output_len; self->proto = 0; self->bin = 0; self->framing = 0; self->frame_start = -1; + self->buf_size = 0; self->fast = 0; self->fast_nesting = 0; self->fix_imports = 0; self->fast_memo = NULL; - self->max_output_len = WRITE_BUF_SIZE; - self->output_len = 0; - self->reducer_override = NULL; - - self->memo = PyMemoTable_New(); - if (self->memo == NULL) { - Py_DECREF(self); - return NULL; - } - self->output_buffer = PyBytes_FromStringAndSize(NULL, - self->max_output_len); - if (self->output_buffer == NULL) { - Py_DECREF(self); - return NULL; - } + self->buffer_callback = NULL; PyObject_GC_Track(self); return self; + +error: + PyMem_Free(memo); + Py_XDECREF(output_buffer); + return NULL; } static int @@ -1628,14 +1635,31 @@ _Unpickler_MemoCleanup(UnpicklerObject *self) static UnpicklerObject * _Unpickler_New(PyObject *module) { - UnpicklerObject *self; + const int MEMO_SIZE = 32; + PyObject **memo = _Unpickler_NewMemo(MEMO_SIZE); + if (memo == NULL) { + return NULL; + } + PickleState *st = _Pickle_GetState(module); + PyObject *stack = Pdata_New(st); + if (stack == NULL) { + goto error; + } - self = PyObject_GC_New(UnpicklerObject, st->Unpickler_Type); - if (self == NULL) - return NULL; + UnpicklerObject *self = PyObject_GC_New(UnpicklerObject, + st->Unpickler_Type); + if (self == NULL) { + goto error; + } + self->stack = (Pdata *)stack; + self->memo = memo; + self->memo_size = MEMO_SIZE; + self->memo_len = 0; self->pers_func = NULL; + self->pers_func_self = NULL; + memset(&self->buffer, 0, sizeof(Py_buffer)); self->input_buffer = NULL; self->input_line = NULL; self->input_len = 0; @@ -1653,22 +1677,14 @@ _Unpickler_New(PyObject *module) self->marks_size = 0; self->proto = 0; self->fix_imports = 0; - memset(&self->buffer, 0, sizeof(Py_buffer)); - self->memo_size = 32; - self->memo_len = 0; - self->memo = _Unpickler_NewMemo(self->memo_size); - if (self->memo == NULL) { - Py_DECREF(self); - return NULL; - } - self->stack = (Pdata *)Pdata_New(st); - if (self->stack == NULL) { - Py_DECREF(self); - return NULL; - } PyObject_GC_Track(self); return self; + +error: + PyMem_Free(memo); + Py_XDECREF(stack); + return NULL; } /* Returns -1 (with an exception set) on failure, 0 on success. This may @@ -1678,25 +1694,30 @@ _Unpickler_SetInputStream(UnpicklerObject *self, PyObject *file) { /* Optional file methods */ if (_PyObject_LookupAttr(file, &_Py_ID(peek), &self->peek) < 0) { - return -1; + goto error; } if (_PyObject_LookupAttr(file, &_Py_ID(readinto), &self->readinto) < 0) { - return -1; + goto error; + } + if (_PyObject_LookupAttr(file, &_Py_ID(read), &self->read) < 0) { + goto error; + } + if (_PyObject_LookupAttr(file, &_Py_ID(readline), &self->readline) < 0) { + goto error; } - (void)_PyObject_LookupAttr(file, &_Py_ID(read), &self->read); - (void)_PyObject_LookupAttr(file, &_Py_ID(readline), &self->readline); if (!self->readline || !self->read) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "file must have 'read' and 'readline' attributes"); - } - Py_CLEAR(self->read); - Py_CLEAR(self->readinto); - Py_CLEAR(self->readline); - Py_CLEAR(self->peek); - return -1; + PyErr_SetString(PyExc_TypeError, + "file must have 'read' and 'readline' attributes"); + goto error; } return 0; + +error: + Py_CLEAR(self->read); + Py_CLEAR(self->readinto); + Py_CLEAR(self->readline); + Py_CLEAR(self->peek); + return -1; } /* Returns -1 (with an exception set) on failure, 0 on success. This may diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index d36a911a57c02c..db5be842b8a35c 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -210,6 +210,7 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, PyObject *item; PyLockStatus r; PY_TIMEOUT_T microseconds; + PyThreadState *tstate = PyThreadState_Get(); if (block == 0) { /* Non-blocking */ @@ -253,7 +254,7 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, Py_END_ALLOW_THREADS } - if (r == PY_LOCK_INTR && Py_MakePendingCalls() < 0) { + if (r == PY_LOCK_INTR && _PyEval_MakePendingCalls(tstate) < 0) { return NULL; } if (r == PY_LOCK_FAILURE) { diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index f8a1a05a318889..2f1c7324a0fa46 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -1549,10 +1549,12 @@ _sre_template_impl(PyObject *module, PyObject *pattern, PyObject *template) for (Py_ssize_t i = 0; i < n; i++) { Py_ssize_t index = PyLong_AsSsize_t(PyList_GET_ITEM(template, 2*i+1)); if (index == -1 && PyErr_Occurred()) { + Py_SET_SIZE(self, i); Py_DECREF(self); return NULL; } if (index < 0) { + Py_SET_SIZE(self, i); goto bad_template; } self->items[i].index = index; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index de90a4a168d2ed..5f90584a99c080 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2996,7 +2996,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) /*[clinic end generated code: output=2cf0d7a0741b6bd1 input=8d58a805b95fc534]*/ { PySSLContext *self; - long options; + uint64_t options; const SSL_METHOD *method = NULL; SSL_CTX *ctx = NULL; X509_VERIFY_PARAM *params; @@ -3594,20 +3594,32 @@ PyDoc_STRVAR(PySSLContext_security_level_doc, "The current security level"); static PyObject * get_options(PySSLContext *self, void *c) { - return PyLong_FromLong(SSL_CTX_get_options(self->ctx)); + uint64_t options = SSL_CTX_get_options(self->ctx); + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(options)); + return PyLong_FromUnsignedLongLong(options); } static int set_options(PySSLContext *self, PyObject *arg, void *c) { - long new_opts, opts, set, clear; - long opt_no = ( + PyObject *new_opts_obj; + unsigned long long new_opts_arg; + uint64_t new_opts, opts, clear, set; + uint64_t opt_no = ( SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1 | SSL_OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3 ); - if (!PyArg_Parse(arg, "l", &new_opts)) + if (!PyArg_Parse(arg, "O!", &PyLong_Type, &new_opts_obj)) { return -1; + } + new_opts_arg = PyLong_AsUnsignedLongLong(new_opts_obj); + if (new_opts_arg == (unsigned long long)-1 && PyErr_Occurred()) { + return -1; + } + Py_BUILD_ASSERT(sizeof(new_opts) >= sizeof(new_opts_arg)); + new_opts = (uint64_t)new_opts_arg; + opts = SSL_CTX_get_options(self->ctx); clear = opts & ~new_opts; set = ~opts & new_opts; @@ -3621,8 +3633,9 @@ set_options(PySSLContext *self, PyObject *arg, void *c) if (clear) { SSL_CTX_clear_options(self->ctx, clear); } - if (set) + if (set) { SSL_CTX_set_options(self->ctx, set); + } return 0; } @@ -5731,10 +5744,24 @@ sslmodule_init_socketapi(PyObject *module) return 0; } + static int -sslmodule_init_constants(PyObject *m) +sslmodule_add_option(PyObject *m, const char *name, uint64_t value) { + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(value)); + PyObject *obj = PyLong_FromUnsignedLongLong(value); + if (obj == NULL) { + return -1; + } + int res = PyModule_AddObjectRef(m, name, obj); + Py_DECREF(obj); + return res; +} + +static int +sslmodule_init_constants(PyObject *m) +{ PyModule_AddStringConstant(m, "_DEFAULT_CIPHERS", PY_SSL_DEFAULT_CIPHER_STRING); @@ -5854,46 +5881,47 @@ sslmodule_init_constants(PyObject *m) PyModule_AddIntConstant(m, "PROTOCOL_TLSv1_2", PY_SSL_VERSION_TLS1_2); +#define ADD_OPTION(NAME, VALUE) if (sslmodule_add_option(m, NAME, (VALUE)) < 0) return -1 + /* protocol options */ - PyModule_AddIntConstant(m, "OP_ALL", - SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); - PyModule_AddIntConstant(m, "OP_NO_SSLv2", SSL_OP_NO_SSLv2); - PyModule_AddIntConstant(m, "OP_NO_SSLv3", SSL_OP_NO_SSLv3); - PyModule_AddIntConstant(m, "OP_NO_TLSv1", SSL_OP_NO_TLSv1); - PyModule_AddIntConstant(m, "OP_NO_TLSv1_1", SSL_OP_NO_TLSv1_1); - PyModule_AddIntConstant(m, "OP_NO_TLSv1_2", SSL_OP_NO_TLSv1_2); + ADD_OPTION("OP_ALL", SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); + ADD_OPTION("OP_NO_SSLv2", SSL_OP_NO_SSLv2); + ADD_OPTION("OP_NO_SSLv3", SSL_OP_NO_SSLv3); + ADD_OPTION("OP_NO_TLSv1", SSL_OP_NO_TLSv1); + ADD_OPTION("OP_NO_TLSv1_1", SSL_OP_NO_TLSv1_1); + ADD_OPTION("OP_NO_TLSv1_2", SSL_OP_NO_TLSv1_2); #ifdef SSL_OP_NO_TLSv1_3 - PyModule_AddIntConstant(m, "OP_NO_TLSv1_3", SSL_OP_NO_TLSv1_3); + ADD_OPTION("OP_NO_TLSv1_3", SSL_OP_NO_TLSv1_3); #else - PyModule_AddIntConstant(m, "OP_NO_TLSv1_3", 0); + ADD_OPTION("OP_NO_TLSv1_3", 0); #endif - PyModule_AddIntConstant(m, "OP_CIPHER_SERVER_PREFERENCE", + ADD_OPTION("OP_CIPHER_SERVER_PREFERENCE", SSL_OP_CIPHER_SERVER_PREFERENCE); - PyModule_AddIntConstant(m, "OP_SINGLE_DH_USE", SSL_OP_SINGLE_DH_USE); - PyModule_AddIntConstant(m, "OP_NO_TICKET", SSL_OP_NO_TICKET); - PyModule_AddIntConstant(m, "OP_LEGACY_SERVER_CONNECT", + ADD_OPTION("OP_SINGLE_DH_USE", SSL_OP_SINGLE_DH_USE); + ADD_OPTION("OP_NO_TICKET", SSL_OP_NO_TICKET); + ADD_OPTION("OP_LEGACY_SERVER_CONNECT", SSL_OP_LEGACY_SERVER_CONNECT); #ifdef SSL_OP_SINGLE_ECDH_USE - PyModule_AddIntConstant(m, "OP_SINGLE_ECDH_USE", SSL_OP_SINGLE_ECDH_USE); + ADD_OPTION("OP_SINGLE_ECDH_USE", SSL_OP_SINGLE_ECDH_USE); #endif #ifdef SSL_OP_NO_COMPRESSION - PyModule_AddIntConstant(m, "OP_NO_COMPRESSION", + ADD_OPTION("OP_NO_COMPRESSION", SSL_OP_NO_COMPRESSION); #endif #ifdef SSL_OP_ENABLE_MIDDLEBOX_COMPAT - PyModule_AddIntConstant(m, "OP_ENABLE_MIDDLEBOX_COMPAT", + ADD_OPTION("OP_ENABLE_MIDDLEBOX_COMPAT", SSL_OP_ENABLE_MIDDLEBOX_COMPAT); #endif #ifdef SSL_OP_NO_RENEGOTIATION - PyModule_AddIntConstant(m, "OP_NO_RENEGOTIATION", + ADD_OPTION("OP_NO_RENEGOTIATION", SSL_OP_NO_RENEGOTIATION); #endif #ifdef SSL_OP_IGNORE_UNEXPECTED_EOF - PyModule_AddIntConstant(m, "OP_IGNORE_UNEXPECTED_EOF", + ADD_OPTION("OP_IGNORE_UNEXPECTED_EOF", SSL_OP_IGNORE_UNEXPECTED_EOF); #endif #ifdef SSL_OP_ENABLE_KTLS - PyModule_AddIntConstant(m, "OP_ENABLE_KTLS", SSL_OP_ENABLE_KTLS); + ADD_OPTION("OP_ENABLE_KTLS", SSL_OP_ENABLE_KTLS); #endif #ifdef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT @@ -6001,15 +6029,21 @@ sslmodule_init_errorcodes(PyObject *module) errcode = error_codes; while (errcode->mnemonic != NULL) { - PyObject *mnemo, *key; - mnemo = PyUnicode_FromString(errcode->mnemonic); - key = Py_BuildValue("ii", errcode->library, errcode->reason); - if (mnemo == NULL || key == NULL) + PyObject *mnemo = PyUnicode_FromString(errcode->mnemonic); + if (mnemo == NULL) { return -1; - if (PyDict_SetItem(state->err_codes_to_names, key, mnemo)) + } + PyObject *key = Py_BuildValue("ii", errcode->library, errcode->reason); + if (key == NULL) { + Py_DECREF(mnemo); return -1; + } + int rc = PyDict_SetItem(state->err_codes_to_names, key, mnemo); Py_DECREF(key); Py_DECREF(mnemo); + if (rc < 0) { + return -1; + } errcode++; } diff --git a/Modules/_testcapi/code.c b/Modules/_testcapi/code.c index 84c668cd6b3b00..cadaf5eb94692e 100644 --- a/Modules/_testcapi/code.c +++ b/Modules/_testcapi/code.c @@ -9,7 +9,7 @@ get_code_extra_index(PyInterpreterState* interp) { PyObject *interp_dict = PyInterpreterState_GetDict(interp); // borrowed assert(interp_dict); // real users would handle missing dict... somehow - PyObject *index_obj = PyDict_GetItemString(interp_dict, key); // borrowed + PyObject *index_obj = _PyDict_GetItemStringWithError(interp_dict, key); // borrowed Py_ssize_t index = 0; if (!index_obj) { if (PyErr_Occurred()) { diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 565ab570a42bde..c124871e433431 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -744,6 +744,12 @@ static PyType_Spec HeapCTypeMetaclassCustomNew_spec = { HeapCTypeMetaclassCustomNew_slots }; +static PyType_Spec HeapCTypeMetaclassNullNew_spec = { + .name = "_testcapi.HeapCTypeMetaclassNullNew", + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .slots = empty_type_slots +}; + typedef struct { PyObject_HEAD @@ -1231,6 +1237,13 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { } PyModule_AddObject(m, "HeapCTypeMetaclassCustomNew", HeapCTypeMetaclassCustomNew); + PyObject *HeapCTypeMetaclassNullNew = PyType_FromMetaclass( + &PyType_Type, m, &HeapCTypeMetaclassNullNew_spec, (PyObject *) &PyType_Type); + if (HeapCTypeMetaclassNullNew == NULL) { + return -1; + } + PyModule_AddObject(m, "HeapCTypeMetaclassNullNew", HeapCTypeMetaclassNullNew); + PyObject *HeapCCollection = PyType_FromMetaclass( NULL, m, &HeapCCollection_spec, NULL); if (HeapCCollection == NULL) { diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 73929eaffc676d..b4d7bf82d73c83 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -376,6 +376,22 @@ unicode_readchar(PyObject *self, PyObject *args) return PyLong_FromUnsignedLong(result); } +/* Test PyUnicode_FromEncodedObject() */ +static PyObject * +unicode_fromencodedobject(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "Oz|z", &obj, &encoding, &errors)) { + return NULL; + } + + NULLABLE(obj); + return PyUnicode_FromEncodedObject(obj, encoding, errors); +} + /* Test PyUnicode_FromObject() */ static PyObject * unicode_fromobject(PyObject *self, PyObject *arg) @@ -669,6 +685,78 @@ unicode_transformdecimalandspacetoascii(PyObject *self, PyObject *arg) return _PyUnicode_TransformDecimalAndSpaceToASCII(arg); } +/* Test PyUnicode_Decode() */ +static PyObject * +unicode_decode(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t size; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#z|z", &s, &size, &encoding, &errors)) + return NULL; + + return PyUnicode_Decode(s, size, encoding, errors); +} + +/* Test PyUnicode_AsEncodedString() */ +static PyObject * +unicode_asencodedstring(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "Oz|z", &unicode, &encoding, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_AsEncodedString(unicode, encoding, errors); +} + +/* Test PyUnicode_BuildEncodingMap() */ +static PyObject * +unicode_buildencodingmap(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_BuildEncodingMap(arg); +} + +/* Test PyUnicode_DecodeUTF7() */ +static PyObject * +unicode_decodeutf7(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF7(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF7Stateful() */ +static PyObject * +unicode_decodeutf7stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF7Stateful(data, size, errors, &consumed); + if (!result) { + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + /* Test PyUnicode_DecodeUTF8() */ static PyObject * unicode_decodeutf8(PyObject *self, PyObject *args) @@ -703,6 +791,387 @@ unicode_decodeutf8stateful(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, consumed); } +/* Test PyUnicode_AsUTF8String() */ +static PyObject * +unicode_asutf8string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF8String(arg); +} + +/* Test PyUnicode_DecodeUTF32() */ +static PyObject * +unicode_decodeutf32(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF32(data, size, errors, &byteorder); + if (!result) { + return NULL; + } + return Py_BuildValue("(iN)", byteorder, result); +} + +/* Test PyUnicode_DecodeUTF32Stateful() */ +static PyObject * +unicode_decodeutf32stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder; + Py_ssize_t consumed; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, &consumed); + if (!result) { + return NULL; + } + return Py_BuildValue("(iNn)", byteorder, result, consumed); +} + +/* Test PyUnicode_AsUTF32String() */ +static PyObject * +unicode_asutf32string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF32String(arg); +} + +/* Test PyUnicode_DecodeUTF16() */ +static PyObject * +unicode_decodeutf16(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = 0; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF16(data, size, errors, &byteorder); + if (!result) { + return NULL; + } + return Py_BuildValue("(iN)", byteorder, result); +} + +/* Test PyUnicode_DecodeUTF16Stateful() */ +static PyObject * +unicode_decodeutf16stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder; + Py_ssize_t consumed; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, &consumed); + if (!result) { + return NULL; + } + return Py_BuildValue("(iNn)", byteorder, result, consumed); +} + +/* Test PyUnicode_AsUTF16String() */ +static PyObject * +unicode_asutf16string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF16String(arg); +} + +/* Test PyUnicode_DecodeUnicodeEscape() */ +static PyObject * +unicode_decodeunicodeescape(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUnicodeEscape(data, size, errors); +} + +/* Test PyUnicode_AsUnicodeEscapeString() */ +static PyObject * +unicode_asunicodeescapestring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUnicodeEscapeString(arg); +} + +static PyObject * +unicode_decoderawunicodeescape(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeRawUnicodeEscape(data, size, errors); +} + +/* Test PyUnicode_AsRawUnicodeEscapeString() */ +static PyObject * +unicode_asrawunicodeescapestring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsRawUnicodeEscapeString(arg); +} + +static PyObject * +unicode_decodelatin1(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLatin1(data, size, errors); +} + +/* Test PyUnicode_AsLatin1String() */ +static PyObject * +unicode_aslatin1string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsLatin1String(arg); +} + +/* Test PyUnicode_DecodeASCII() */ +static PyObject * +unicode_decodeascii(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeASCII(data, size, errors); +} + +/* Test PyUnicode_AsASCIIString() */ +static PyObject * +unicode_asasciistring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsASCIIString(arg); +} + +/* Test PyUnicode_DecodeCharmap() */ +static PyObject * +unicode_decodecharmap(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + PyObject *mapping; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#O|z", &data, &size, &mapping, &errors)) + return NULL; + + NULLABLE(mapping); + return PyUnicode_DecodeCharmap(data, size, mapping, errors); +} + +/* Test PyUnicode_AsCharmapString() */ +static PyObject * +unicode_ascharmapstring(PyObject *self, PyObject *args) +{ + PyObject *unicode; + PyObject *mapping; + + if (!PyArg_ParseTuple(args, "OO", &unicode, &mapping)) + return NULL; + + NULLABLE(unicode); + NULLABLE(mapping); + return PyUnicode_AsCharmapString(unicode, mapping); +} + +#ifdef MS_WINDOWS + +/* Test PyUnicode_DecodeMBCS() */ +static PyObject * +unicode_decodembcs(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeMBCS(data, size, errors); +} + +/* Test PyUnicode_DecodeMBCSStateful() */ +static PyObject * +unicode_decodembcsstateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeMBCSStateful(data, size, errors, &consumed); + if (!result) { + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_DecodeCodePageStateful() */ +static PyObject * +unicode_decodecodepagestateful(PyObject *self, PyObject *args) +{ + int code_page; + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &code_page, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeCodePageStateful(code_page, data, size, errors, &consumed); + if (!result) { + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_AsMBCSString() */ +static PyObject * +unicode_asmbcsstring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsMBCSString(arg); +} + +/* Test PyUnicode_EncodeCodePage() */ +static PyObject * +unicode_encodecodepage(PyObject *self, PyObject *args) +{ + int code_page; + PyObject *unicode; + const char *errors; + + if (!PyArg_ParseTuple(args, "iO|z", &code_page, &unicode, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_EncodeCodePage(code_page, unicode, errors); +} + +#endif /* MS_WINDOWS */ + +/* Test PyUnicode_DecodeLocaleAndSize() */ +static PyObject * +unicode_decodelocaleandsize(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLocaleAndSize(data, size, errors); +} + +/* Test PyUnicode_DecodeLocale() */ +static PyObject * +unicode_decodelocale(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLocale(data, errors); +} + +/* Test PyUnicode_EncodeLocale() */ +static PyObject * +unicode_encodelocale(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *errors; + + if (!PyArg_ParseTuple(args, "O|z", &unicode, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_EncodeLocale(unicode, errors); +} + +/* Test PyUnicode_DecodeFSDefault() */ +static PyObject * +unicode_decodefsdefault(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "y#", &data, &size)) + return NULL; + + return PyUnicode_DecodeFSDefault(data); +} + +/* Test PyUnicode_DecodeFSDefaultAndSize() */ +static PyObject * +unicode_decodefsdefaultandsize(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "y#|n", &data, &size, &size)) + return NULL; + + return PyUnicode_DecodeFSDefaultAndSize(data, size); +} + +/* Test PyUnicode_EncodeFSDefault() */ +static PyObject * +unicode_encodefsdefault(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_EncodeFSDefault(arg); +} + /* Test PyUnicode_Concat() */ static PyObject * unicode_concat(PyObject *self, PyObject *args) @@ -1528,6 +1997,7 @@ static PyMethodDef TestMethods[] = { {"unicode_substring", unicode_substring, METH_VARARGS}, {"unicode_getlength", unicode_getlength, METH_O}, {"unicode_readchar", unicode_readchar, METH_VARARGS}, + {"unicode_fromencodedobject",unicode_fromencodedobject, METH_VARARGS}, {"unicode_fromobject", unicode_fromobject, METH_O}, {"unicode_interninplace", unicode_interninplace, METH_O}, {"unicode_internfromstring", unicode_internfromstring, METH_O}, @@ -1542,9 +2012,44 @@ static PyMethodDef TestMethods[] = { {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, + {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, + {"unicode_decode", unicode_decode, METH_VARARGS}, + {"unicode_asencodedstring", unicode_asencodedstring, METH_VARARGS}, + {"unicode_buildencodingmap", unicode_buildencodingmap, METH_O}, + {"unicode_decodeutf7", unicode_decodeutf7, METH_VARARGS}, + {"unicode_decodeutf7stateful",unicode_decodeutf7stateful, METH_VARARGS}, {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, - {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, + {"unicode_asutf8string", unicode_asutf8string, METH_O}, + {"unicode_decodeutf16", unicode_decodeutf16, METH_VARARGS}, + {"unicode_decodeutf16stateful",unicode_decodeutf16stateful, METH_VARARGS}, + {"unicode_asutf16string", unicode_asutf16string, METH_O}, + {"unicode_decodeutf32", unicode_decodeutf32, METH_VARARGS}, + {"unicode_decodeutf32stateful",unicode_decodeutf32stateful, METH_VARARGS}, + {"unicode_asutf32string", unicode_asutf32string, METH_O}, + {"unicode_decodeunicodeescape",unicode_decodeunicodeescape, METH_VARARGS}, + {"unicode_asunicodeescapestring",unicode_asunicodeescapestring,METH_O}, + {"unicode_decoderawunicodeescape",unicode_decoderawunicodeescape,METH_VARARGS}, + {"unicode_asrawunicodeescapestring",unicode_asrawunicodeescapestring,METH_O}, + {"unicode_decodelatin1", unicode_decodelatin1, METH_VARARGS}, + {"unicode_aslatin1string", unicode_aslatin1string, METH_O}, + {"unicode_decodeascii", unicode_decodeascii, METH_VARARGS}, + {"unicode_asasciistring", unicode_asasciistring, METH_O}, + {"unicode_decodecharmap", unicode_decodecharmap, METH_VARARGS}, + {"unicode_ascharmapstring", unicode_ascharmapstring, METH_VARARGS}, +#ifdef MS_WINDOWS + {"unicode_decodembcs", unicode_decodembcs, METH_VARARGS}, + {"unicode_decodembcsstateful",unicode_decodembcsstateful, METH_VARARGS}, + {"unicode_decodecodepagestateful",unicode_decodecodepagestateful,METH_VARARGS}, + {"unicode_asmbcsstring", unicode_asmbcsstring, METH_O}, + {"unicode_encodecodepage", unicode_encodecodepage, METH_VARARGS}, +#endif /* MS_WINDOWS */ + {"unicode_decodelocaleandsize",unicode_decodelocaleandsize, METH_VARARGS}, + {"unicode_decodelocale", unicode_decodelocale, METH_VARARGS}, + {"unicode_encodelocale", unicode_encodelocale, METH_VARARGS}, + {"unicode_decodefsdefault", unicode_decodefsdefault, METH_VARARGS}, + {"unicode_decodefsdefaultandsize",unicode_decodefsdefaultandsize,METH_VARARGS}, + {"unicode_encodefsdefault", unicode_encodefsdefault, METH_O}, {"unicode_transformdecimalandspacetoascii", unicode_transformdecimalandspacetoascii, METH_O}, {"unicode_concat", unicode_concat, METH_VARARGS}, {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index d2c71fb401d36a..7167943fffab39 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -432,9 +432,9 @@ allocate_too_many_code_watchers(PyObject *self, PyObject *args) // Test function watchers -#define NUM_FUNC_WATCHERS 2 -static PyObject *pyfunc_watchers[NUM_FUNC_WATCHERS]; -static int func_watcher_ids[NUM_FUNC_WATCHERS] = {-1, -1}; +#define NUM_TEST_FUNC_WATCHERS 2 +static PyObject *pyfunc_watchers[NUM_TEST_FUNC_WATCHERS]; +static int func_watcher_ids[NUM_TEST_FUNC_WATCHERS] = {-1, -1}; static PyObject * get_id(PyObject *obj) @@ -508,7 +508,7 @@ second_func_watcher_callback(PyFunction_WatchEvent event, return call_pyfunc_watcher(pyfunc_watchers[1], event, func, new_value); } -static PyFunction_WatchCallback func_watcher_callbacks[NUM_FUNC_WATCHERS] = { +static PyFunction_WatchCallback func_watcher_callbacks[NUM_TEST_FUNC_WATCHERS] = { first_func_watcher_callback, second_func_watcher_callback }; @@ -533,26 +533,25 @@ add_func_watcher(PyObject *self, PyObject *func) return NULL; } int idx = -1; - for (int i = 0; i < NUM_FUNC_WATCHERS; i++) { + for (int i = 0; i < NUM_TEST_FUNC_WATCHERS; i++) { if (func_watcher_ids[i] == -1) { idx = i; break; } } if (idx == -1) { - PyErr_SetString(PyExc_RuntimeError, "no free watchers"); - return NULL; - } - PyObject *result = PyLong_FromLong(idx); - if (result == NULL) { + PyErr_SetString(PyExc_RuntimeError, "no free test watchers"); return NULL; } func_watcher_ids[idx] = PyFunction_AddWatcher(func_watcher_callbacks[idx]); if (func_watcher_ids[idx] < 0) { - Py_DECREF(result); return NULL; } pyfunc_watchers[idx] = Py_NewRef(func); + PyObject *result = PyLong_FromLong(func_watcher_ids[idx]); + if (result == NULL) { + return NULL; + } return result; } @@ -569,7 +568,7 @@ clear_func_watcher(PyObject *self, PyObject *watcher_id_obj) return NULL; } int idx = -1; - for (int i = 0; i < NUM_FUNC_WATCHERS; i++) { + for (int i = 0; i < NUM_TEST_FUNC_WATCHERS; i++) { if (func_watcher_ids[i] == wid) { idx = i; break; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b8ad00a0197421..6523734131c2de 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -640,6 +640,30 @@ test_get_type_qualname(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } +static PyObject * +test_get_type_dict(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + /* Test for PyType_GetDict */ + + // Assert ints have a `to_bytes` method + PyObject *long_dict = PyType_GetDict(&PyLong_Type); + assert(long_dict); + assert(PyDict_GetItemString(long_dict, "to_bytes")); // borrowed ref + Py_DECREF(long_dict); + + // Make a new type, add an attribute to it and assert it's there + PyObject *HeapTypeNameType = PyType_FromSpec(&HeapTypeNameType_Spec); + assert(HeapTypeNameType); + assert(PyObject_SetAttrString( + HeapTypeNameType, "new_attr", Py_NewRef(Py_None)) >= 0); + PyObject *type_dict = PyType_GetDict((PyTypeObject*)HeapTypeNameType); + assert(type_dict); + assert(PyDict_GetItemString(type_dict, "new_attr")); // borrowed ref + Py_DECREF(HeapTypeNameType); + Py_DECREF(type_dict); + Py_RETURN_NONE; +} + static PyObject * pyobject_repr_from_null(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -1426,7 +1450,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) int allow_threads = -1; int allow_daemon_threads = -1; int check_multi_interp_extensions = -1; - int own_gil = -1; + int gil = -1; int r; PyThreadState *substate, *mainstate; /* only initialise 'cflags.cf_flags' to test backwards compatibility */ @@ -1439,15 +1463,15 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) "allow_threads", "allow_daemon_threads", "check_multi_interp_extensions", - "own_gil", + "gil", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$ppppppp:run_in_subinterp_with_config", kwlist, + "s$ppppppi:run_in_subinterp_with_config", kwlist, &code, &use_main_obmalloc, &allow_fork, &allow_exec, &allow_threads, &allow_daemon_threads, &check_multi_interp_extensions, - &own_gil)) { + &gil)) { return NULL; } if (use_main_obmalloc < 0) { @@ -1466,8 +1490,8 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_ValueError, "missing allow_threads"); return NULL; } - if (own_gil < 0) { - PyErr_SetString(PyExc_ValueError, "missing own_gil"); + if (gil < 0) { + PyErr_SetString(PyExc_ValueError, "missing gil"); return NULL; } if (allow_daemon_threads < 0) { @@ -1490,7 +1514,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) .allow_threads = allow_threads, .allow_daemon_threads = allow_daemon_threads, .check_multi_interp_extensions = check_multi_interp_extensions, - .own_gil = own_gil, + .gil = gil, }; PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); if (PyStatus_Exception(status)) { @@ -3347,6 +3371,7 @@ static PyMethodDef TestMethods[] = { {"test_get_statictype_slots", test_get_statictype_slots, METH_NOARGS}, {"test_get_type_name", test_get_type_name, METH_NOARGS}, {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, + {"test_get_type_dict", test_get_type_dict, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, #ifndef MS_WINDOWS {"_spawn_pthread_waiter", spawn_pthread_waiter, METH_NOARGS}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 8267dbf6779017..98f99e4d66cf1f 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -15,14 +15,17 @@ #include "frameobject.h" #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() +#include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble +#include "pycore_ceval.h" // _PyEval_AddPendingCall #include "pycore_fileutils.h" // _Py_normpath #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() -#include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() +#include "pycore_interpreteridobject.h" // _PyInterpreterID_LookUp() +#include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() #include "pycore_pystate.h" // _PyThreadState_GET() #include "osdefs.h" // MAXPATHLEN @@ -439,6 +442,118 @@ test_edit_cost(PyObject *self, PyObject *Py_UNUSED(args)) } +static int +check_bytes_find(const char *haystack0, const char *needle0, + int offset, Py_ssize_t expected) +{ + Py_ssize_t len_haystack = strlen(haystack0); + Py_ssize_t len_needle = strlen(needle0); + Py_ssize_t result_1 = _PyBytes_Find(haystack0, len_haystack, + needle0, len_needle, offset); + if (result_1 != expected) { + PyErr_Format(PyExc_AssertionError, + "Incorrect result_1: '%s' in '%s' (offset=%zd)", + needle0, haystack0, offset); + return -1; + } + // Allocate new buffer with no NULL terminator. + char *haystack = PyMem_Malloc(len_haystack); + if (haystack == NULL) { + PyErr_NoMemory(); + return -1; + } + char *needle = PyMem_Malloc(len_needle); + if (needle == NULL) { + PyMem_Free(haystack); + PyErr_NoMemory(); + return -1; + } + memcpy(haystack, haystack0, len_haystack); + memcpy(needle, needle0, len_needle); + Py_ssize_t result_2 = _PyBytes_Find(haystack, len_haystack, + needle, len_needle, offset); + PyMem_Free(haystack); + PyMem_Free(needle); + if (result_2 != expected) { + PyErr_Format(PyExc_AssertionError, + "Incorrect result_2: '%s' in '%s' (offset=%zd)", + needle0, haystack0, offset); + return -1; + } + return 0; +} + +static int +check_bytes_find_large(Py_ssize_t len_haystack, Py_ssize_t len_needle, + const char *needle) +{ + char *zeros = PyMem_RawCalloc(len_haystack, 1); + if (zeros == NULL) { + PyErr_NoMemory(); + return -1; + } + Py_ssize_t res = _PyBytes_Find(zeros, len_haystack, needle, len_needle, 0); + PyMem_RawFree(zeros); + if (res != -1) { + PyErr_Format(PyExc_AssertionError, + "check_bytes_find_large(%zd, %zd) found %zd", + len_haystack, len_needle, res); + return -1; + } + return 0; +} + +static PyObject * +test_bytes_find(PyObject *self, PyObject *Py_UNUSED(args)) +{ + #define CHECK(H, N, O, E) do { \ + if (check_bytes_find(H, N, O, E) < 0) { \ + return NULL; \ + } \ + } while (0) + + CHECK("", "", 0, 0); + CHECK("Python", "", 0, 0); + CHECK("Python", "", 3, 3); + CHECK("Python", "", 6, 6); + CHECK("Python", "yth", 0, 1); + CHECK("ython", "yth", 1, 1); + CHECK("thon", "yth", 2, -1); + CHECK("Python", "thon", 0, 2); + CHECK("ython", "thon", 1, 2); + CHECK("thon", "thon", 2, 2); + CHECK("hon", "thon", 3, -1); + CHECK("Pytho", "zz", 0, -1); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ab", 0, -1); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ba", 0, -1); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bb", 0, -1); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", "ab", 0, 30); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaba", "ba", 0, 30); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaabb", "bb", 0, 30); + #undef CHECK + + // Hunt for segfaults + // n, m chosen here so that (n - m) % (m + 1) == 0 + // This would make default_find in fastsearch.h access haystack[n]. + if (check_bytes_find_large(2048, 2, "ab") < 0) { + return NULL; + } + if (check_bytes_find_large(4096, 16, "0123456789abcdef") < 0) { + return NULL; + } + if (check_bytes_find_large(8192, 2, "ab") < 0) { + return NULL; + } + if (check_bytes_find_large(16384, 4, "abcd") < 0) { + return NULL; + } + if (check_bytes_find_large(32768, 2, "ab") < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + static PyObject * normalize_path(PyObject *self, PyObject *filename) { @@ -822,6 +937,120 @@ iframe_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(PyUnstable_InterpreterFrame_GetLasti(f)); } + +static int _pending_callback(void *arg) +{ + /* we assume the argument is callable object to which we own a reference */ + PyObject *callable = (PyObject *)arg; + PyObject *r = PyObject_CallNoArgs(callable); + Py_DECREF(callable); + Py_XDECREF(r); + return r != NULL ? 0 : -1; +} + +/* The following requests n callbacks to _pending_callback. It can be + * run from any python thread. + */ +static PyObject * +pending_threadfunc(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *callable; + int ensure_added = 0; + static char *kwlist[] = {"", "ensure_added", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O|$p:pending_threadfunc", kwlist, + &callable, &ensure_added)) + { + return NULL; + } + PyInterpreterState *interp = PyInterpreterState_Get(); + + /* create the reference for the callbackwhile we hold the lock */ + Py_INCREF(callable); + + int r; + Py_BEGIN_ALLOW_THREADS + r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); + Py_END_ALLOW_THREADS + if (r < 0) { + /* unsuccessful add */ + if (!ensure_added) { + Py_DECREF(callable); + Py_RETURN_FALSE; + } + do { + Py_BEGIN_ALLOW_THREADS + r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); + Py_END_ALLOW_THREADS + } while (r < 0); + } + + Py_RETURN_TRUE; +} + + +static struct { + int64_t interpid; +} pending_identify_result; + +static int +_pending_identify_callback(void *arg) +{ + PyThread_type_lock mutex = (PyThread_type_lock)arg; + assert(pending_identify_result.interpid == -1); + PyThreadState *tstate = PyThreadState_Get(); + pending_identify_result.interpid = PyInterpreterState_GetID(tstate->interp); + PyThread_release_lock(mutex); + return 0; +} + +static PyObject * +pending_identify(PyObject *self, PyObject *args) +{ + PyObject *interpid; + if (!PyArg_ParseTuple(args, "O:pending_identify", &interpid)) { + return NULL; + } + PyInterpreterState *interp = _PyInterpreterID_LookUp(interpid); + if (interp == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, "interpreter not found"); + } + return NULL; + } + + pending_identify_result.interpid = -1; + + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return NULL; + } + PyThread_acquire_lock(mutex, WAIT_LOCK); + /* It gets released in _pending_identify_callback(). */ + + int r; + do { + Py_BEGIN_ALLOW_THREADS + r = _PyEval_AddPendingCall(interp, + &_pending_identify_callback, (void *)mutex, + 0); + Py_END_ALLOW_THREADS + } while (r < 0); + + /* Wait for the pending call to complete. */ + PyThread_acquire_lock(mutex, WAIT_LOCK); + PyThread_release_lock(mutex); + PyThread_free_lock(mutex); + + PyObject *res = PyLong_FromLongLong(pending_identify_result.interpid); + pending_identify_result.interpid = -1; + if (res == NULL) { + return NULL; + } + return res; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -834,6 +1063,7 @@ static PyMethodDef module_functions[] = { {"reset_path_config", test_reset_path_config, METH_NOARGS}, {"test_atomic_funcs", test_atomic_funcs, METH_NOARGS}, {"test_edit_cost", test_edit_cost, METH_NOARGS}, + {"test_bytes_find", test_bytes_find, METH_NOARGS}, {"normalize_path", normalize_path, METH_O, NULL}, {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL}, {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS}, @@ -850,6 +1080,9 @@ static PyMethodDef module_functions[] = { {"iframe_getcode", iframe_getcode, METH_O, NULL}, {"iframe_getline", iframe_getline, METH_O, NULL}, {"iframe_getlasti", iframe_getlasti, METH_O, NULL}, + {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), + METH_VARARGS | METH_KEYWORDS}, + {"pending_identify", pending_identify, METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index b6f878e07526db..c553d039462af0 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -81,6 +81,7 @@ lock_dealloc(lockobject *self) static PyLockStatus acquire_timed(PyThread_type_lock lock, _PyTime_t timeout) { + PyThreadState *tstate = _PyThreadState_GET(); _PyTime_t endtime = 0; if (timeout > 0) { endtime = _PyDeadline_Init(timeout); @@ -103,7 +104,7 @@ acquire_timed(PyThread_type_lock lock, _PyTime_t timeout) /* Run signal handlers if we were interrupted. Propagate * exceptions from signal handlers, such as KeyboardInterrupt, by * passing up PY_LOCK_INTR. */ - if (Py_MakePendingCalls() < 0) { + if (_PyEval_MakePendingCalls(tstate) < 0) { return PY_LOCK_INTR; } diff --git a/Modules/_winapi.c b/Modules/_winapi.c index bbc9facd227c9e..d6d2f4a6a9b103 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -796,6 +796,17 @@ getenvironment(PyObject* environment) } envsize = PyList_GET_SIZE(keys); + + if (envsize == 0) { + // A environment block must be terminated by two null characters -- + // one for the last string and one for the block. + buffer = PyMem_Calloc(2, sizeof(wchar_t)); + if (!buffer) { + PyErr_NoMemory(); + } + goto cleanup; + } + if (PyList_GET_SIZE(values) != envsize) { PyErr_SetString(PyExc_RuntimeError, "environment changed size during iteration"); @@ -869,7 +880,8 @@ getenvironment(PyObject* environment) *p++ = L'\0'; assert(p == end); - error: +cleanup: +error: Py_XDECREF(keys); Py_XDECREF(values); return buffer; diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index f94bbec8e0bb3c..6680820d8e61bc 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -739,10 +739,12 @@ array_richcompare(PyObject *v, PyObject *w, int op) k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { vi = getarrayitem(v, i); + if (vi == NULL) { + return NULL; + } wi = getarrayitem(w, i); - if (vi == NULL || wi == NULL) { - Py_XDECREF(vi); - Py_XDECREF(wi); + if (wi == NULL) { + Py_DECREF(vi); return NULL; } k = PyObject_RichCompareBool(vi, wi, Py_EQ); diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6bde9939eaa2ca..1ffadccb916253 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -343,12 +343,17 @@ mmap_gfind(mmap_object *self, Py_ssize_t res; CHECK_VALID_OR_RELEASE(NULL, view); - if (reverse) { + if (end < start) { + res = -1; + } + else if (reverse) { + assert(0 <= start && start <= end && end <= self->size); res = _PyBytes_ReverseFind( self->data + start, end - start, view.buf, view.len, start); } else { + assert(0 <= start && start <= end && end <= self->size); res = _PyBytes_Find( self->data + start, end - start, view.buf, view.len, start); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f5e653dd023aef..81c9990220893d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6414,7 +6414,7 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) { Py_ssize_t i, pos, envc; PyObject *keys=NULL, *vals=NULL; - PyObject *key, *val, *key2, *val2, *keyval; + PyObject *key2, *val2, *keyval; EXECV_CHAR **envlist; i = PyMapping_Size(env); @@ -6439,10 +6439,14 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) } for (pos = 0; pos < i; pos++) { - key = PyList_GetItem(keys, pos); - val = PyList_GetItem(vals, pos); - if (!key || !val) + PyObject *key = PyList_GetItem(keys, pos); // Borrowed ref. + if (key == NULL) { goto error; + } + PyObject *val = PyList_GetItem(vals, pos); // Borrowed ref. + if (val == NULL) { + goto error; + } #if defined(HAVE_WEXECV) || defined(HAVE_WSPAWNV) if (!PyUnicode_FSDecoder(key, &key2)) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 92f594ab63ea2a..b21360419d6a14 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1775,14 +1775,18 @@ add_error(PyObject *errors_module, PyObject *codes_dict, static int add_errors_module(PyObject *mod) { + // add_submodule() returns a borrowed ref. PyObject *errors_module = add_submodule(mod, MODULE_NAME ".errors"); if (errors_module == NULL) { return -1; } PyObject *codes_dict = PyDict_New(); + if (codes_dict == NULL) { + return -1; + } PyObject *rev_codes_dict = PyDict_New(); - if (codes_dict == NULL || rev_codes_dict == NULL) { + if (rev_codes_dict == NULL) { goto error; } @@ -1803,17 +1807,17 @@ add_errors_module(PyObject *mod) goto error; } - if (PyModule_AddObject(errors_module, "codes", Py_NewRef(codes_dict)) < 0) { - Py_DECREF(codes_dict); + int rc = PyModule_AddObjectRef(errors_module, "codes", codes_dict); + Py_CLEAR(codes_dict); + if (rc < 0) { goto error; } - Py_CLEAR(codes_dict); - if (PyModule_AddObject(errors_module, "messages", Py_NewRef(rev_codes_dict)) < 0) { - Py_DECREF(rev_codes_dict); + rc = PyModule_AddObjectRef(errors_module, "messages", rev_codes_dict); + Py_CLEAR(rev_codes_dict); + if (rc < 0) { goto error; } - Py_CLEAR(rev_codes_dict); return 0; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 2350236ad46b25..00ea4343735dab 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -314,7 +314,8 @@ trip_signal(int sig_num) still use it for this exceptional case. */ _PyEval_AddPendingCall(interp, report_wakeup_send_error, - (void *)(intptr_t) last_error); + (void *)(intptr_t) last_error, + 1); } } } @@ -333,7 +334,8 @@ trip_signal(int sig_num) still use it for this exceptional case. */ _PyEval_AddPendingCall(interp, report_wakeup_write_error, - (void *)(intptr_t)errno); + (void *)(intptr_t)errno, + 1); } } } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index abbf3eeb16c35c..f3a978c86c3606 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1274,8 +1274,25 @@ _PyBytes_Find(const char *haystack, Py_ssize_t len_haystack, const char *needle, Py_ssize_t len_needle, Py_ssize_t offset) { - return stringlib_find(haystack, len_haystack, - needle, len_needle, offset); + assert(len_haystack >= 0); + assert(len_needle >= 0); + // Extra checks because stringlib_find accesses haystack[len_haystack]. + if (len_needle == 0) { + return offset; + } + if (len_needle > len_haystack) { + return -1; + } + assert(len_haystack >= 1); + Py_ssize_t res = stringlib_find(haystack, len_haystack - 1, + needle, len_needle, offset); + if (res == -1) { + Py_ssize_t last_align = len_haystack - len_needle; + if (memcmp(haystack + last_align, needle, len_needle) == 0) { + return offset + last_align; + } + } + return res; } Py_ssize_t diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 9b54c610581174..d47ca731a10c87 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2030,6 +2030,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, co_code, co_filename, co_name, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags) < 0) { + Py_XDECREF(code); return NULL; } diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 7bec7395cc7f0b..f376ff24386a37 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -207,22 +207,21 @@ BaseException_add_note(PyObject *self, PyObject *note) return NULL; } - if (!PyObject_HasAttr(self, &_Py_ID(__notes__))) { - PyObject *new_notes = PyList_New(0); - if (new_notes == NULL) { + PyObject *notes; + if (_PyObject_LookupAttr(self, &_Py_ID(__notes__), ¬es) < 0) { + return NULL; + } + if (notes == NULL) { + notes = PyList_New(0); + if (notes == NULL) { return NULL; } - if (PyObject_SetAttr(self, &_Py_ID(__notes__), new_notes) < 0) { - Py_DECREF(new_notes); + if (PyObject_SetAttr(self, &_Py_ID(__notes__), notes) < 0) { + Py_DECREF(notes); return NULL; } - Py_DECREF(new_notes); } - PyObject *notes = PyObject_GetAttr(self, &_Py_ID(__notes__)); - if (notes == NULL) { - return NULL; - } - if (!PyList_Check(notes)) { + else if (!PyList_Check(notes)) { Py_DECREF(notes); PyErr_SetString(PyExc_TypeError, "Cannot add note: __notes__ is not a list"); return NULL; @@ -941,11 +940,11 @@ exceptiongroup_subset( PyException_SetContext(eg, PyException_GetContext(orig)); PyException_SetCause(eg, PyException_GetCause(orig)); - if (PyObject_HasAttr(orig, &_Py_ID(__notes__))) { - PyObject *notes = PyObject_GetAttr(orig, &_Py_ID(__notes__)); - if (notes == NULL) { - goto error; - } + PyObject *notes; + if (_PyObject_LookupAttr(orig, &_Py_ID(__notes__), ¬es) < 0) { + goto error; + } + if (notes) { if (PySequence_Check(notes)) { /* Make a copy so the parts have independent notes lists. */ PyObject *notes_copy = PySequence_List(notes); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2c90a6b71311ca..e62cfab80b2e35 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1199,15 +1199,28 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, return 1; } -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) + +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden) { /* Merge fast locals into f->f_locals */ PyObject *locals = frame->f_locals; if (locals == NULL) { locals = frame->f_locals = PyDict_New(); if (locals == NULL) { - return -1; + return NULL; + } + } + PyObject *hidden = NULL; + + /* If include_hidden, "hidden" fast locals (from inlined comprehensions in + module/class scopes) will be included in the returned dict, but not in + frame->f_locals; the returned dict will be a modified copy. Non-hidden + locals will still be updated in frame->f_locals. */ + if (include_hidden) { + hidden = PyDict_New(); + if (hidden == NULL) { + return NULL; } } @@ -1223,6 +1236,11 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); if (kind & CO_FAST_HIDDEN) { + if (include_hidden && value != NULL) { + if (PyObject_SetItem(hidden, name, value) != 0) { + goto error; + } + } continue; } if (value == NULL) { @@ -1231,16 +1249,53 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) PyErr_Clear(); } else { - return -1; + goto error; } } } else { if (PyObject_SetItem(locals, name, value) != 0) { - return -1; + goto error; } } } + + if (include_hidden && PyDict_Size(hidden)) { + PyObject *innerlocals = PyDict_New(); + if (innerlocals == NULL) { + goto error; + } + if (PyDict_Merge(innerlocals, locals, 1) != 0) { + Py_DECREF(innerlocals); + goto error; + } + if (PyDict_Merge(innerlocals, hidden, 1) != 0) { + Py_DECREF(innerlocals); + goto error; + } + locals = innerlocals; + } + else { + Py_INCREF(locals); + } + Py_CLEAR(hidden); + + return locals; + + error: + Py_XDECREF(hidden); + return NULL; +} + + +int +_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) +{ + PyObject *locals = _PyFrame_GetLocals(frame, 0); + if (locals == NULL) { + return -1; + } + Py_DECREF(locals); return 0; } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 753038600aa858..f43e3a2787b846 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -106,9 +106,14 @@ PyFunction_ClearWatcher(int watcher_id) PyFunctionObject * _PyFunction_FromConstructor(PyFrameConstructor *constr) { + PyObject *module = Py_XNewRef(PyDict_GetItemWithError(constr->fc_globals, &_Py_ID(__name__))); + if (!module && PyErr_Occurred()) { + return NULL; + } PyFunctionObject *op = PyObject_GC_New(PyFunctionObject, &PyFunction_Type); if (op == NULL) { + Py_XDECREF(module); return NULL; } op->func_globals = Py_NewRef(constr->fc_globals); @@ -122,10 +127,7 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_doc = Py_NewRef(Py_None); op->func_dict = NULL; op->func_weakreflist = NULL; - op->func_module = Py_XNewRef(PyDict_GetItem(op->func_globals, &_Py_ID(__name__))); - if (!op->func_module) { - PyErr_Clear(); - } + op->func_module = module; op->func_annotations = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 888cb16edd1b46..117b4e8dfb960a 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -121,6 +121,36 @@ ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) return err; } +static int +ga_repr_items_list(_PyUnicodeWriter *writer, PyObject *p) +{ + assert(PyList_CheckExact(p)); + + Py_ssize_t len = PyList_GET_SIZE(p); + + if (_PyUnicodeWriter_WriteASCIIString(writer, "[", 1) < 0) { + return -1; + } + + for (Py_ssize_t i = 0; i < len; i++) { + if (i > 0) { + if (_PyUnicodeWriter_WriteASCIIString(writer, ", ", 2) < 0) { + return -1; + } + } + PyObject *item = PyList_GET_ITEM(p, i); + if (ga_repr_item(writer, item) < 0) { + return -1; + } + } + + if (_PyUnicodeWriter_WriteASCIIString(writer, "]", 1) < 0) { + return -1; + } + + return 0; +} + static PyObject * ga_repr(PyObject *self) { @@ -148,7 +178,13 @@ ga_repr(PyObject *self) } } PyObject *p = PyTuple_GET_ITEM(alias->args, i); - if (ga_repr_item(&writer, p) < 0) { + if (PyList_CheckExact(p)) { + // Looks like we are working with ParamSpec's list of type args: + if (ga_repr_items_list(&writer, p) < 0) { + goto error; + } + } + else if (ga_repr_item(&writer, p) < 0) { goto error; } } diff --git a/Objects/object.c b/Objects/object.c index ece0c5e21e77fa..b8bdf459201d9b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1704,13 +1704,15 @@ _dir_locals(void) PyObject *names; PyObject *locals; - locals = PyEval_GetLocals(); + locals = _PyEval_GetFrameLocals(); if (locals == NULL) return NULL; names = PyMapping_Keys(locals); - if (!names) + Py_DECREF(locals); + if (!names) { return NULL; + } if (!PyList_Check(names)) { PyErr_Format(PyExc_TypeError, "dir(): expected keys() of locals to be a list, " @@ -1722,7 +1724,6 @@ _dir_locals(void) Py_DECREF(names); return NULL; } - /* the locals don't need to be DECREF'd */ return names; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b6771d3004df91..6662379515f95d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -238,6 +238,13 @@ _PyType_GetDict(PyTypeObject *self) return lookup_tp_dict(self); } +PyObject * +PyType_GetDict(PyTypeObject *self) +{ + PyObject *dict = lookup_tp_dict(self); + return _Py_XNewRef(dict); +} + static inline void set_tp_dict(PyTypeObject *self, PyObject *dict) { @@ -1507,11 +1514,14 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) static PyObject * type_get_type_params(PyTypeObject *type, void *context) { - PyObject *params = PyDict_GetItem(lookup_tp_dict(type), &_Py_ID(__type_params__)); + PyObject *params = PyDict_GetItemWithError(lookup_tp_dict(type), &_Py_ID(__type_params__)); if (params) { return Py_NewRef(params); } + if (PyErr_Occurred()) { + return NULL; + } return PyTuple_New(0); } @@ -4236,7 +4246,7 @@ _PyType_FromMetaclass_impl( metaclass); goto finally; } - if (metaclass->tp_new != PyType_Type.tp_new) { + if (metaclass->tp_new && metaclass->tp_new != PyType_Type.tp_new) { if (_allow_tp_new) { if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 406a6eb76e3a8a..97f39136996fcc 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -500,7 +500,7 @@ PyType_Spec typevar_spec = { .name = "typing.TypeVar", .basicsize = sizeof(typevarobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE - | Py_TPFLAGS_MANAGED_DICT, + | Py_TPFLAGS_MANAGED_DICT | Py_TPFLAGS_MANAGED_WEAKREF, .slots = typevar_slots, }; @@ -647,7 +647,8 @@ static PyType_Slot paramspecargs_slots[] = { PyType_Spec paramspecargs_spec = { .name = "typing.ParamSpecArgs", .basicsize = sizeof(paramspecattrobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_MANAGED_WEAKREF, .slots = paramspecargs_slots, }; @@ -726,7 +727,8 @@ static PyType_Slot paramspeckwargs_slots[] = { PyType_Spec paramspeckwargs_spec = { .name = "typing.ParamSpecKwargs", .basicsize = sizeof(paramspecattrobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_MANAGED_WEAKREF, .slots = paramspeckwargs_slots, }; @@ -1007,7 +1009,7 @@ PyType_Spec paramspec_spec = { .name = "typing.ParamSpec", .basicsize = sizeof(paramspecobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE - | Py_TPFLAGS_MANAGED_DICT, + | Py_TPFLAGS_MANAGED_DICT | Py_TPFLAGS_MANAGED_WEAKREF, .slots = paramspec_slots, }; @@ -1228,7 +1230,7 @@ PyType_Spec typevartuple_spec = { .name = "typing.TypeVarTuple", .basicsize = sizeof(typevartupleobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_DICT - | Py_TPFLAGS_HAVE_GC, + | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_WEAKREF, .slots = typevartuple_slots, }; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 6f25f91c0ff28d..a0be85718119de 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -7934,25 +7934,30 @@ PyUnicode_BuildEncodingMap(PyObject* string) if (need_dict) { PyObject *result = PyDict_New(); - PyObject *key, *value; if (!result) return NULL; for (i = 0; i < length; i++) { - key = PyLong_FromLong(PyUnicode_READ(kind, data, i)); - value = PyLong_FromLong(i); - if (!key || !value) - goto failed1; - if (PyDict_SetItem(result, key, value) == -1) - goto failed1; + Py_UCS4 c = PyUnicode_READ(kind, data, i); + PyObject *key = PyLong_FromLong(c); + if (key == NULL) { + Py_DECREF(result); + return NULL; + } + PyObject *value = PyLong_FromLong(i); + if (value == NULL) { + Py_DECREF(key); + Py_DECREF(result); + return NULL; + } + int rc = PyDict_SetItem(result, key, value); Py_DECREF(key); Py_DECREF(value); + if (rc < 0) { + Py_DECREF(result); + return NULL; + } } return result; - failed1: - Py_XDECREF(key); - Py_XDECREF(value); - Py_DECREF(result); - return NULL; } /* Create a three-level trie */ diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 534a9ebcfd151a..77b9594799ea79 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,7 +53,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1u +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.9 set libraries=%libraries% sqlite-3.42.0.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 @@ -77,7 +77,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1u +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.9 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/openssl.props b/PCbuild/openssl.props index 7071bb57c06cd7..5fd708b211e42d 100644 --- a/PCbuild/openssl.props +++ b/PCbuild/openssl.props @@ -10,10 +10,10 @@ - <_DLLSuffix>-1_1 + <_DLLSuffix>-3 <_DLLSuffix Condition="$(Platform) == 'ARM'">$(_DLLSuffix)-arm <_DLLSuffix Condition="$(Platform) == 'ARM64'">$(_DLLSuffix)-arm64 - $(_DLLSuffix) + $(_DLLSuffix) <_SSLDLL Include="$(opensslOutDir)\libcrypto$(_DLLSuffix).dll" /> diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 36c4c269d05da9..fd928cafd77aaa 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -25,7 +25,7 @@ <_VCToolsVersion>$([System.Version]::Parse(`$(VCToolsVersion)`).Major).$([System.Version]::Parse(`$(VCToolsVersion)`).Minor) - true + true diff --git a/PCbuild/python.props b/PCbuild/python.props index 68052ef668aa6c..d3586235c82652 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -74,8 +74,8 @@ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include - $(ExternalsDir)openssl-1.1.1u\ - $(ExternalsDir)openssl-bin-1.1.1u\$(ArchName)\ + $(ExternalsDir)openssl-3.0.9\ + $(ExternalsDir)openssl-bin-3.0.9\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.2.13\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 784d3ce6a60660..2ab4537b99882e 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -169,7 +169,7 @@ _lzma Homepage: https://tukaani.org/xz/ _ssl - Python wrapper for version 1.1.1u of the OpenSSL secure sockets + Python wrapper for version 3.0 of the OpenSSL secure sockets library, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 107066817ba6b0..af07adeca8812b 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -104,8 +104,9 @@ <_LicenseSources Include="$(PySourcePath)LICENSE; $(PySourcePath)PC\crtlicense.txt; $(bz2Dir)LICENSE; - $(opensslOutDir)LICENSE; $(libffiDir)LICENSE;" /> + <_LicenseSources Include="$(opensslOutDir)LICENSE.txt" Condition="Exists('$(opensslOutDir)LICENSE.txt')" /> + <_LicenseSources Include="$(opensslOutDir)LICENSE" Condition="!Exists('$(opensslOutDir)LICENSE.txt')" /> <_LicenseSources Include="$(tcltkDir)tcllicense.terms; $(tcltkDir)tklicense.terms; $(tcltkDir)tixlicense.terms" Condition="$(IncludeTkinter)" /> diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 93632a09f0959b..0d154867276c36 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -148,5 +148,5 @@ module Python type_param = TypeVar(identifier name, expr? bound) | ParamSpec(identifier name) | TypeVarTuple(identifier name) - attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) } diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 9f5135380db1b2..e68a9cac25908c 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1003,6 +1003,18 @@ _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, in if (!spec) { return NULL; } + + // This is needed to keep compatibility with 3.11, where an empty format spec is parsed + // as an *empty* JoinedStr node, instead of having an empty constant in it. + if (asdl_seq_LEN(spec) == 1) { + expr_ty e = asdl_seq_GET(spec, 0); + if (e->kind == Constant_kind + && PyUnicode_Check(e->v.Constant.value) + && PyUnicode_GetLength(e->v.Constant.value) == 0) { + spec = _Py_asdl_expr_seq_new(0, arena); + } + } + expr_ty res = _PyAST_JoinedStr(spec, lineno, col_offset, end_lineno, end_col_offset, p->arena); if (!res) { return NULL; @@ -1231,7 +1243,7 @@ _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq // Fstring stuff static expr_ty -_PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant) { +_PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* token) { assert(PyUnicode_CheckExact(constant->v.Constant.value)); const char* bstr = PyUnicode_AsUTF8(constant->v.Constant.value); @@ -1247,7 +1259,7 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant) { } is_raw = is_raw || strchr(bstr, '\\') == NULL; - PyObject *str = _PyPegen_decode_string(p, is_raw, bstr, len, NULL); + PyObject *str = _PyPegen_decode_string(p, is_raw, bstr, len, token); if (str == NULL) { _Pypegen_raise_decode_error(p); return NULL; @@ -1321,7 +1333,7 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b for (Py_ssize_t i = 0; i < n_items; i++) { expr_ty item = asdl_seq_GET(expr, i); if (item->kind == Constant_kind) { - item = _PyPegen_decode_fstring_part(p, is_raw, item); + item = _PyPegen_decode_fstring_part(p, is_raw, item, b); if (item == NULL) { return NULL; } diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index cb312796f89e04..d4763ea260a5a2 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -601,6 +601,7 @@ def visitProduct(self, prod, name): args = [f.name for f in prod.fields] args.extend([a.name for a in prod.attributes]) self.emit("*out = %s(%s);" % (ast_func_name(name), self.buildArgs(args)), 1) + self.emit("if (*out == NULL) goto failed;", 1) self.emit("return 0;", 1) self.emit("failed:", 0) self.emit("Py_XDECREF(tmp);", 1) diff --git a/Parser/parser.c b/Parser/parser.c index 006ee297974a61..f2ea8f59b00567 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -532,14 +532,14 @@ static char *soft_keywords[] = { #define _gather_207_type 1451 #define _loop0_210_type 1452 #define _gather_209_type 1453 -#define _tmp_211_type 1454 -#define _loop0_212_type 1455 -#define _loop1_213_type 1456 -#define _tmp_214_type 1457 -#define _loop0_215_type 1458 -#define _loop1_216_type 1459 -#define _tmp_217_type 1460 -#define _tmp_218_type 1461 +#define _loop0_212_type 1454 +#define _gather_211_type 1455 +#define _tmp_213_type 1456 +#define _loop0_214_type 1457 +#define _loop1_215_type 1458 +#define _tmp_216_type 1459 +#define _loop0_217_type 1460 +#define _loop1_218_type 1461 #define _tmp_219_type 1462 #define _tmp_220_type 1463 #define _tmp_221_type 1464 @@ -548,10 +548,10 @@ static char *soft_keywords[] = { #define _tmp_224_type 1467 #define _tmp_225_type 1468 #define _tmp_226_type 1469 -#define _loop0_228_type 1470 -#define _gather_227_type 1471 -#define _tmp_229_type 1472 -#define _tmp_230_type 1473 +#define _tmp_227_type 1470 +#define _tmp_228_type 1471 +#define _loop0_230_type 1472 +#define _gather_229_type 1473 #define _tmp_231_type 1474 #define _tmp_232_type 1475 #define _tmp_233_type 1476 @@ -563,9 +563,9 @@ static char *soft_keywords[] = { #define _tmp_239_type 1482 #define _tmp_240_type 1483 #define _tmp_241_type 1484 -#define _loop0_242_type 1485 +#define _tmp_242_type 1485 #define _tmp_243_type 1486 -#define _tmp_244_type 1487 +#define _loop0_244_type 1487 #define _tmp_245_type 1488 #define _tmp_246_type 1489 #define _tmp_247_type 1490 @@ -596,6 +596,8 @@ static char *soft_keywords[] = { #define _tmp_272_type 1515 #define _tmp_273_type 1516 #define _tmp_274_type 1517 +#define _tmp_275_type 1518 +#define _tmp_276_type 1519 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -1051,14 +1053,14 @@ static asdl_seq *_loop0_208_rule(Parser *p); static asdl_seq *_gather_207_rule(Parser *p); static asdl_seq *_loop0_210_rule(Parser *p); static asdl_seq *_gather_209_rule(Parser *p); -static void *_tmp_211_rule(Parser *p); static asdl_seq *_loop0_212_rule(Parser *p); -static asdl_seq *_loop1_213_rule(Parser *p); -static void *_tmp_214_rule(Parser *p); -static asdl_seq *_loop0_215_rule(Parser *p); -static asdl_seq *_loop1_216_rule(Parser *p); -static void *_tmp_217_rule(Parser *p); -static void *_tmp_218_rule(Parser *p); +static asdl_seq *_gather_211_rule(Parser *p); +static void *_tmp_213_rule(Parser *p); +static asdl_seq *_loop0_214_rule(Parser *p); +static asdl_seq *_loop1_215_rule(Parser *p); +static void *_tmp_216_rule(Parser *p); +static asdl_seq *_loop0_217_rule(Parser *p); +static asdl_seq *_loop1_218_rule(Parser *p); static void *_tmp_219_rule(Parser *p); static void *_tmp_220_rule(Parser *p); static void *_tmp_221_rule(Parser *p); @@ -1067,10 +1069,10 @@ static void *_tmp_223_rule(Parser *p); static void *_tmp_224_rule(Parser *p); static void *_tmp_225_rule(Parser *p); static void *_tmp_226_rule(Parser *p); -static asdl_seq *_loop0_228_rule(Parser *p); -static asdl_seq *_gather_227_rule(Parser *p); -static void *_tmp_229_rule(Parser *p); -static void *_tmp_230_rule(Parser *p); +static void *_tmp_227_rule(Parser *p); +static void *_tmp_228_rule(Parser *p); +static asdl_seq *_loop0_230_rule(Parser *p); +static asdl_seq *_gather_229_rule(Parser *p); static void *_tmp_231_rule(Parser *p); static void *_tmp_232_rule(Parser *p); static void *_tmp_233_rule(Parser *p); @@ -1082,9 +1084,9 @@ static void *_tmp_238_rule(Parser *p); static void *_tmp_239_rule(Parser *p); static void *_tmp_240_rule(Parser *p); static void *_tmp_241_rule(Parser *p); -static asdl_seq *_loop0_242_rule(Parser *p); +static void *_tmp_242_rule(Parser *p); static void *_tmp_243_rule(Parser *p); -static void *_tmp_244_rule(Parser *p); +static asdl_seq *_loop0_244_rule(Parser *p); static void *_tmp_245_rule(Parser *p); static void *_tmp_246_rule(Parser *p); static void *_tmp_247_rule(Parser *p); @@ -1115,6 +1117,8 @@ static void *_tmp_271_rule(Parser *p); static void *_tmp_272_rule(Parser *p); static void *_tmp_273_rule(Parser *p); static void *_tmp_274_rule(Parser *p); +static void *_tmp_275_rule(Parser *p); +static void *_tmp_276_rule(Parser *p); // file: statements? $ @@ -22648,7 +22652,7 @@ invalid_group_rule(Parser *p) return _res; } -// invalid_import: 'import' dotted_name 'from' dotted_name +// invalid_import: 'import' ','.dotted_name+ 'from' dotted_name static void * invalid_import_rule(Parser *p) { @@ -22662,27 +22666,27 @@ invalid_import_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'import' dotted_name 'from' dotted_name + { // 'import' ','.dotted_name+ 'from' dotted_name if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' dotted_name 'from' dotted_name")); + D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); + asdl_seq * _gather_203_var; Token * _keyword; Token * a; expr_ty dotted_name_var; - expr_ty dotted_name_var_1; if ( (a = _PyPegen_expect_token(p, 607)) // token='import' && - (dotted_name_var = dotted_name_rule(p)) // dotted_name + (_gather_203_var = _gather_203_rule(p)) // ','.dotted_name+ && (_keyword = _PyPegen_expect_token(p, 608)) // token='from' && - (dotted_name_var_1 = dotted_name_rule(p)) // dotted_name + (dotted_name_var = dotted_name_rule(p)) // dotted_name ) { - D(fprintf(stderr, "%*c+ invalid_import[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import' dotted_name 'from' dotted_name")); + D(fprintf(stderr, "%*c+ invalid_import[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "Did you mean to use 'from ... import ...' instead?" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -22693,7 +22697,7 @@ invalid_import_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_import[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'import' dotted_name 'from' dotted_name")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); } _res = NULL; done: @@ -22773,7 +22777,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "ASYNC? 'with' ','.(expression ['as' star_target])+ NEWLINE")); - asdl_seq * _gather_203_var; + asdl_seq * _gather_205_var; Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22783,7 +22787,7 @@ invalid_with_stmt_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 615)) // token='with' && - (_gather_203_var = _gather_203_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_205_var = _gather_205_rule(p)) // ','.(expression ['as' star_target])+ && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -22807,7 +22811,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "ASYNC? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE")); - asdl_seq * _gather_205_var; + asdl_seq * _gather_207_var; Token * _keyword; Token * _literal; Token * _literal_1; @@ -22823,7 +22827,7 @@ invalid_with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_205_var = _gather_205_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_207_var = _gather_207_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22873,7 +22877,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "ASYNC? 'with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT")); - asdl_seq * _gather_207_var; + asdl_seq * _gather_209_var; Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22884,7 +22888,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (a = _PyPegen_expect_token(p, 615)) // token='with' && - (_gather_207_var = _gather_207_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_209_var = _gather_209_rule(p)) // ','.(expression ['as' star_target])+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22912,7 +22916,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "ASYNC? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT")); - asdl_seq * _gather_209_var; + asdl_seq * _gather_211_var; Token * _literal; Token * _literal_1; Token * _literal_2; @@ -22929,7 +22933,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_209_var = _gather_209_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_211_var = _gather_211_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -23027,7 +23031,7 @@ invalid_try_stmt_rule(Parser *p) && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, _tmp_211_rule, p) + _PyPegen_lookahead(0, _tmp_213_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); @@ -23052,8 +23056,8 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_212_var; - asdl_seq * _loop1_213_var; + asdl_seq * _loop0_214_var; + asdl_seq * _loop1_215_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -23064,9 +23068,9 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_212_var = _loop0_212_rule(p)) // block* + (_loop0_214_var = _loop0_214_rule(p)) // block* && - (_loop1_213_var = _loop1_213_rule(p)) // except_block+ + (_loop1_215_var = _loop1_215_rule(p)) // except_block+ && (a = _PyPegen_expect_token(p, 637)) // token='except' && @@ -23074,7 +23078,7 @@ invalid_try_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_214_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_216_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23101,8 +23105,8 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_215_var; - asdl_seq * _loop1_216_var; + asdl_seq * _loop0_217_var; + asdl_seq * _loop1_218_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -23111,13 +23115,13 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_215_var = _loop0_215_rule(p)) // block* + (_loop0_217_var = _loop0_217_rule(p)) // block* && - (_loop1_216_var = _loop1_216_rule(p)) // except_star_block+ + (_loop1_218_var = _loop1_218_rule(p)) // except_star_block+ && (a = _PyPegen_expect_token(p, 637)) // token='except' && - (_opt_var = _tmp_217_rule(p), !p->error_indicator) // [expression ['as' NAME]] + (_opt_var = _tmp_219_rule(p), !p->error_indicator) // [expression ['as' NAME]] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23185,7 +23189,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var_1 = _tmp_218_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_220_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23223,7 +23227,7 @@ invalid_except_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var_1 = _tmp_219_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_221_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23275,14 +23279,14 @@ invalid_except_stmt_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); Token * _literal; - void *_tmp_220_var; + void *_tmp_222_var; Token * a; if ( (a = _PyPegen_expect_token(p, 637)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_220_var = _tmp_220_rule(p)) // NEWLINE | ':' + (_tmp_222_var = _tmp_222_rule(p)) // NEWLINE | ':' ) { D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); @@ -23389,7 +23393,7 @@ invalid_except_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_221_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_223_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23484,7 +23488,7 @@ invalid_except_star_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_222_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_224_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23853,7 +23857,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_223_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_225_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -24347,7 +24351,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_2 = _tmp_224_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_2 = _tmp_226_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24407,7 +24411,7 @@ invalid_class_def_raw_rule(Parser *p) && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_225_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var = _tmp_227_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24442,7 +24446,7 @@ invalid_class_def_raw_rule(Parser *p) && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_226_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var = _tmp_228_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24493,11 +24497,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_227_var; + asdl_seq * _gather_229_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_227_var = _gather_227_rule(p)) // ','.double_starred_kvpair+ + (_gather_229_var = _gather_229_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -24505,7 +24509,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_227_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_229_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -24558,7 +24562,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_229_rule, p) + _PyPegen_lookahead(1, _tmp_231_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24669,7 +24673,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_230_rule, p) + _PyPegen_lookahead(1, _tmp_232_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24887,7 +24891,7 @@ invalid_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, _tmp_231_rule, p) + _PyPegen_lookahead(0, _tmp_233_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !(yield_expr | star_expressions)")); @@ -24910,13 +24914,13 @@ invalid_replacement_field_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); Token * _literal; - void *_tmp_232_var; + void *_tmp_234_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_232_var = _tmp_232_rule(p)) // yield_expr | star_expressions + (_tmp_234_var = _tmp_234_rule(p)) // yield_expr | star_expressions && - _PyPegen_lookahead(0, _tmp_233_rule, p) + _PyPegen_lookahead(0, _tmp_235_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); @@ -24940,15 +24944,15 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); Token * _literal; Token * _literal_1; - void *_tmp_234_var; + void *_tmp_236_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_234_var = _tmp_234_rule(p)) // yield_expr | star_expressions + (_tmp_236_var = _tmp_236_rule(p)) // yield_expr | star_expressions && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, _tmp_235_rule, p) + _PyPegen_lookahead(0, _tmp_237_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); @@ -24973,12 +24977,12 @@ invalid_replacement_field_rule(Parser *p) Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_236_var; + void *_tmp_238_var; void *invalid_conversion_character_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_236_var = _tmp_236_rule(p)) // yield_expr | star_expressions + (_tmp_238_var = _tmp_238_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && @@ -24986,7 +24990,7 @@ invalid_replacement_field_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? invalid_conversion_character")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_236_var, _opt_var, invalid_conversion_character_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_238_var, _opt_var, invalid_conversion_character_var); goto done; } p->mark = _mark; @@ -25004,17 +25008,17 @@ invalid_replacement_field_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_237_var; + void *_tmp_239_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_237_var = _tmp_237_rule(p)) // yield_expr | star_expressions + (_tmp_239_var = _tmp_239_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_238_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_240_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, _tmp_239_rule, p) + _PyPegen_lookahead(0, _tmp_241_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}')")); @@ -25038,24 +25042,24 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_242_var; + asdl_seq * _loop0_244_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_240_var; + void *_tmp_242_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_240_var = _tmp_240_rule(p)) // yield_expr | star_expressions + (_tmp_242_var = _tmp_242_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_241_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_243_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_242_var = _loop0_242_rule(p)) // fstring_format_spec* + (_loop0_244_var = _loop0_244_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25084,15 +25088,15 @@ invalid_replacement_field_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_243_var; + void *_tmp_245_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_243_var = _tmp_243_rule(p)) // yield_expr | star_expressions + (_tmp_245_var = _tmp_245_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_244_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_246_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25140,7 +25144,7 @@ invalid_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, _tmp_245_rule, p) + _PyPegen_lookahead(1, _tmp_247_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -26085,12 +26089,12 @@ _loop1_15_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_246_var; + void *_tmp_248_var; while ( - (_tmp_246_var = _tmp_246_rule(p)) // star_targets '=' + (_tmp_248_var = _tmp_248_rule(p)) // star_targets '=' ) { - _res = _tmp_246_var; + _res = _tmp_248_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26664,12 +26668,12 @@ _loop0_25_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_247_var; + void *_tmp_249_var; while ( - (_tmp_247_var = _tmp_247_rule(p)) // '.' | '...' + (_tmp_249_var = _tmp_249_rule(p)) // '.' | '...' ) { - _res = _tmp_247_var; + _res = _tmp_249_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26732,12 +26736,12 @@ _loop1_26_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_26[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_248_var; + void *_tmp_250_var; while ( - (_tmp_248_var = _tmp_248_rule(p)) // '.' | '...' + (_tmp_250_var = _tmp_250_rule(p)) // '.' | '...' ) { - _res = _tmp_248_var; + _res = _tmp_250_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -27137,12 +27141,12 @@ _loop1_33_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_33[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_249_var; + void *_tmp_251_var; while ( - (_tmp_249_var = _tmp_249_rule(p)) // '@' named_expression NEWLINE + (_tmp_251_var = _tmp_251_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_249_var; + _res = _tmp_251_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30317,12 +30321,12 @@ _loop1_83_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_250_var; + void *_tmp_252_var; while ( - (_tmp_250_var = _tmp_250_rule(p)) // ',' expression + (_tmp_252_var = _tmp_252_rule(p)) // ',' expression ) { - _res = _tmp_250_var; + _res = _tmp_252_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30390,12 +30394,12 @@ _loop1_84_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_84[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_251_var; + void *_tmp_253_var; while ( - (_tmp_251_var = _tmp_251_rule(p)) // ',' star_expression + (_tmp_253_var = _tmp_253_rule(p)) // ',' star_expression ) { - _res = _tmp_251_var; + _res = _tmp_253_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30582,12 +30586,12 @@ _loop1_87_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_252_var; + void *_tmp_254_var; while ( - (_tmp_252_var = _tmp_252_rule(p)) // 'or' conjunction + (_tmp_254_var = _tmp_254_rule(p)) // 'or' conjunction ) { - _res = _tmp_252_var; + _res = _tmp_254_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30655,12 +30659,12 @@ _loop1_88_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_253_var; + void *_tmp_255_var; while ( - (_tmp_253_var = _tmp_253_rule(p)) // 'and' inversion + (_tmp_255_var = _tmp_255_rule(p)) // 'and' inversion ) { - _res = _tmp_253_var; + _res = _tmp_255_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30850,7 +30854,7 @@ _loop0_92_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_254_rule(p)) // slice | starred_expression + (elem = _tmp_256_rule(p)) // slice | starred_expression ) { _res = elem; @@ -30916,7 +30920,7 @@ _gather_91_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_254_rule(p)) // slice | starred_expression + (elem = _tmp_256_rule(p)) // slice | starred_expression && (seq = _loop0_92_rule(p)) // _loop0_92 ) @@ -32471,12 +32475,12 @@ _loop1_115_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); - void *_tmp_255_var; + void *_tmp_257_var; while ( - (_tmp_255_var = _tmp_255_rule(p)) // fstring | string + (_tmp_257_var = _tmp_257_rule(p)) // fstring | string ) { - _res = _tmp_255_var; + _res = _tmp_257_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32786,12 +32790,12 @@ _loop0_120_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_256_var; + void *_tmp_258_var; while ( - (_tmp_256_var = _tmp_256_rule(p)) // 'if' disjunction + (_tmp_258_var = _tmp_258_rule(p)) // 'if' disjunction ) { - _res = _tmp_256_var; + _res = _tmp_258_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32854,12 +32858,12 @@ _loop0_121_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_257_var; + void *_tmp_259_var; while ( - (_tmp_257_var = _tmp_257_rule(p)) // 'if' disjunction + (_tmp_259_var = _tmp_259_rule(p)) // 'if' disjunction ) { - _res = _tmp_257_var; + _res = _tmp_259_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32987,7 +32991,7 @@ _loop0_124_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_258_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_260_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -33054,7 +33058,7 @@ _gather_123_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_258_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_260_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && (seq = _loop0_124_rule(p)) // _loop0_124 ) @@ -33625,12 +33629,12 @@ _loop0_134_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_259_var; + void *_tmp_261_var; while ( - (_tmp_259_var = _tmp_259_rule(p)) // ',' star_target + (_tmp_261_var = _tmp_261_rule(p)) // ',' star_target ) { - _res = _tmp_259_var; + _res = _tmp_261_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33812,12 +33816,12 @@ _loop1_137_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_260_var; + void *_tmp_262_var; while ( - (_tmp_260_var = _tmp_260_rule(p)) // ',' star_target + (_tmp_262_var = _tmp_262_rule(p)) // ',' star_target ) { - _res = _tmp_260_var; + _res = _tmp_262_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35338,12 +35342,12 @@ _loop0_162_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_261_var; + void *_tmp_263_var; while ( - (_tmp_261_var = _tmp_261_rule(p)) // star_targets '=' + (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' ) { - _res = _tmp_261_var; + _res = _tmp_263_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35406,12 +35410,12 @@ _loop0_163_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_262_var; + void *_tmp_264_var; while ( - (_tmp_262_var = _tmp_262_rule(p)) // star_targets '=' + (_tmp_264_var = _tmp_264_rule(p)) // star_targets '=' ) { - _res = _tmp_262_var; + _res = _tmp_264_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -36454,15 +36458,15 @@ _tmp_179_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_263_var; + void *_tmp_265_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_263_var = _tmp_263_rule(p)) // ')' | '**' + (_tmp_265_var = _tmp_265_rule(p)) // ')' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_263_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); goto done; } p->mark = _mark; @@ -37628,15 +37632,15 @@ _tmp_197_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_264_var; + void *_tmp_266_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_264_var = _tmp_264_rule(p)) // ':' | '**' + (_tmp_266_var = _tmp_266_rule(p)) // ':' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_264_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_266_var); goto done; } p->mark = _mark; @@ -37987,7 +37991,7 @@ _tmp_202_rule(Parser *p) return _res; } -// _loop0_204: ',' (expression ['as' star_target]) +// _loop0_204: ',' dotted_name static asdl_seq * _loop0_204_rule(Parser *p) { @@ -38010,18 +38014,18 @@ _loop0_204_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' (expression ['as' star_target]) + { // ',' dotted_name if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); Token * _literal; - void *elem; + expr_ty elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_265_rule(p)) // expression ['as' star_target] + (elem = dotted_name_rule(p)) // dotted_name ) { _res = elem; @@ -38048,7 +38052,7 @@ _loop0_204_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s _loop0_204[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -38064,7 +38068,7 @@ _loop0_204_rule(Parser *p) return _seq; } -// _gather_203: (expression ['as' star_target]) _loop0_204 +// _gather_203: dotted_name _loop0_204 static asdl_seq * _gather_203_rule(Parser *p) { @@ -38078,27 +38082,27 @@ _gather_203_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_204 + { // dotted_name _loop0_204 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_204")); - void *elem; + D(fprintf(stderr, "%*c> _gather_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_204")); + expr_ty elem; asdl_seq * seq; if ( - (elem = _tmp_265_rule(p)) // expression ['as' star_target] + (elem = dotted_name_rule(p)) // dotted_name && (seq = _loop0_204_rule(p)) // _loop0_204 ) { - D(fprintf(stderr, "%*c+ _gather_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_204")); + D(fprintf(stderr, "%*c+ _gather_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_204")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _gather_203[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_204")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_204")); } _res = NULL; done: @@ -38106,7 +38110,7 @@ _gather_203_rule(Parser *p) return _res; } -// _loop0_206: ',' (expressions ['as' star_target]) +// _loop0_206: ',' (expression ['as' star_target]) static asdl_seq * _loop0_206_rule(Parser *p) { @@ -38129,18 +38133,18 @@ _loop0_206_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' (expressions ['as' star_target]) + { // ',' (expression ['as' star_target]) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_266_rule(p)) // expressions ['as' star_target] + (elem = _tmp_267_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -38167,7 +38171,7 @@ _loop0_206_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s _loop0_206[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -38183,7 +38187,7 @@ _loop0_206_rule(Parser *p) return _seq; } -// _gather_205: (expressions ['as' star_target]) _loop0_206 +// _gather_205: (expression ['as' star_target]) _loop0_206 static asdl_seq * _gather_205_rule(Parser *p) { @@ -38197,27 +38201,27 @@ _gather_205_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_206 + { // (expression ['as' star_target]) _loop0_206 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c> _gather_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_266_rule(p)) // expressions ['as' star_target] + (elem = _tmp_267_rule(p)) // expression ['as' star_target] && (seq = _loop0_206_rule(p)) // _loop0_206 ) { - D(fprintf(stderr, "%*c+ _gather_205[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c+ _gather_205[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _gather_205[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_206")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); } _res = NULL; done: @@ -38225,7 +38229,7 @@ _gather_205_rule(Parser *p) return _res; } -// _loop0_208: ',' (expression ['as' star_target]) +// _loop0_208: ',' (expressions ['as' star_target]) static asdl_seq * _loop0_208_rule(Parser *p) { @@ -38248,18 +38252,18 @@ _loop0_208_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' (expression ['as' star_target]) + { // ',' (expressions ['as' star_target]) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_267_rule(p)) // expression ['as' star_target] + (elem = _tmp_268_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38286,7 +38290,7 @@ _loop0_208_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s _loop0_208[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -38302,7 +38306,7 @@ _loop0_208_rule(Parser *p) return _seq; } -// _gather_207: (expression ['as' star_target]) _loop0_208 +// _gather_207: (expressions ['as' star_target]) _loop0_208 static asdl_seq * _gather_207_rule(Parser *p) { @@ -38316,27 +38320,27 @@ _gather_207_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_208 + { // (expressions ['as' star_target]) _loop0_208 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c> _gather_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_267_rule(p)) // expression ['as' star_target] + (elem = _tmp_268_rule(p)) // expressions ['as' star_target] && (seq = _loop0_208_rule(p)) // _loop0_208 ) { - D(fprintf(stderr, "%*c+ _gather_207[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c+ _gather_207[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _gather_207[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_208")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); } _res = NULL; done: @@ -38344,7 +38348,7 @@ _gather_207_rule(Parser *p) return _res; } -// _loop0_210: ',' (expressions ['as' star_target]) +// _loop0_210: ',' (expression ['as' star_target]) static asdl_seq * _loop0_210_rule(Parser *p) { @@ -38367,18 +38371,18 @@ _loop0_210_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' (expressions ['as' star_target]) + { // ',' (expression ['as' star_target]) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_268_rule(p)) // expressions ['as' star_target] + (elem = _tmp_269_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -38405,7 +38409,7 @@ _loop0_210_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s _loop0_210[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -38421,7 +38425,7 @@ _loop0_210_rule(Parser *p) return _seq; } -// _gather_209: (expressions ['as' star_target]) _loop0_210 +// _gather_209: (expression ['as' star_target]) _loop0_210 static asdl_seq * _gather_209_rule(Parser *p) { @@ -38435,27 +38439,27 @@ _gather_209_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_210 + { // (expression ['as' star_target]) _loop0_210 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c> _gather_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_268_rule(p)) // expressions ['as' star_target] + (elem = _tmp_269_rule(p)) // expression ['as' star_target] && (seq = _loop0_210_rule(p)) // _loop0_210 ) { - D(fprintf(stderr, "%*c+ _gather_209[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c+ _gather_209[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _gather_209[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_210")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); } _res = NULL; done: @@ -38463,65 +38467,7 @@ _gather_209_rule(Parser *p) return _res; } -// _tmp_211: 'except' | 'finally' -static void * -_tmp_211_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // 'except' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); - Token * _keyword; - if ( - (_keyword = _PyPegen_expect_token(p, 637)) // token='except' - ) - { - D(fprintf(stderr, "%*c+ _tmp_211[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); - _res = _keyword; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_211[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); - } - { // 'finally' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); - Token * _keyword; - if ( - (_keyword = _PyPegen_expect_token(p, 633)) // token='finally' - ) - { - D(fprintf(stderr, "%*c+ _tmp_211[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); - _res = _keyword; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_211[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_212: block +// _loop0_212: ',' (expressions ['as' star_target]) static asdl_seq * _loop0_212_rule(Parser *p) { @@ -38544,18 +38490,27 @@ _loop0_212_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // block + { // ',' (expressions ['as' star_target]) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); - asdl_stmt_seq* block_var; + D(fprintf(stderr, "%*c> _loop0_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + Token * _literal; + void *elem; while ( - (block_var = block_rule(p)) // block + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = _tmp_270_rule(p)) // expressions ['as' star_target] ) { - _res = block_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -38573,7 +38528,7 @@ _loop0_212_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s _loop0_212[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -38589,9 +38544,9 @@ _loop0_212_rule(Parser *p) return _seq; } -// _loop1_213: except_block +// _gather_211: (expressions ['as' star_target]) _loop0_212 static asdl_seq * -_loop1_213_rule(Parser *p) +_gather_211_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38601,70 +38556,39 @@ _loop1_213_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // except_block + { // (expressions ['as' star_target]) _loop0_212 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); - excepthandler_ty except_block_var; - while ( - (except_block_var = except_block_rule(p)) // except_block + D(fprintf(stderr, "%*c> _gather_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + void *elem; + asdl_seq * seq; + if ( + (elem = _tmp_270_rule(p)) // expressions ['as' star_target] + && + (seq = _loop0_212_rule(p)) // _loop0_212 ) { - _res = except_block_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _gather_211[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_213[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _gather_211[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _tmp_214: 'as' NAME +// _tmp_213: 'except' | 'finally' static void * -_tmp_214_rule(Parser *p) +_tmp_213_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38676,27 +38600,43 @@ _tmp_214_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'as' NAME + { // 'except' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; - expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 640)) // token='as' - && - (name_var = _PyPegen_name_token(p)) // NAME + (_keyword = _PyPegen_expect_token(p, 637)) // token='except' ) { - D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = _PyPegen_dummy_name(p, _keyword, name_var); + D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); + _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); + } + { // 'finally' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); + Token * _keyword; + if ( + (_keyword = _PyPegen_expect_token(p, 633)) // token='finally' + ) + { + D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); + _res = _keyword; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); } _res = NULL; done: @@ -38704,9 +38644,9 @@ _tmp_214_rule(Parser *p) return _res; } -// _loop0_215: block +// _loop0_214: block static asdl_seq * -_loop0_215_rule(Parser *p) +_loop0_214_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38732,7 +38672,7 @@ _loop0_215_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38755,7 +38695,7 @@ _loop0_215_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_215[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_214[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38772,9 +38712,9 @@ _loop0_215_rule(Parser *p) return _seq; } -// _loop1_216: except_star_block +// _loop1_215: except_block static asdl_seq * -_loop1_216_rule(Parser *p) +_loop1_215_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38795,18 +38735,18 @@ _loop1_216_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // except_star_block + { // except_block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); - excepthandler_ty except_star_block_var; + D(fprintf(stderr, "%*c> _loop1_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); + excepthandler_ty except_block_var; while ( - (except_star_block_var = except_star_block_rule(p)) // except_star_block + (except_block_var = except_block_rule(p)) // except_block ) { - _res = except_star_block_var; + _res = except_block_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -38823,8 +38763,8 @@ _loop1_216_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_216[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); + D(fprintf(stderr, "%*c%s _loop1_215[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -38845,9 +38785,192 @@ _loop1_216_rule(Parser *p) return _seq; } -// _tmp_217: expression ['as' NAME] +// _tmp_216: 'as' NAME static void * -_tmp_217_rule(Parser *p) +_tmp_216_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + p->error_indicator = 1; + PyErr_NoMemory(); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // 'as' NAME + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + Token * _keyword; + expr_ty name_var; + if ( + (_keyword = _PyPegen_expect_token(p, 640)) // token='as' + && + (name_var = _PyPegen_name_token(p)) // NAME + ) + { + D(fprintf(stderr, "%*c+ _tmp_216[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + _res = _PyPegen_dummy_name(p, _keyword, name_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_216[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _loop0_217: block +static asdl_seq * +_loop0_217_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + p->error_indicator = 1; + PyErr_NoMemory(); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // block + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop0_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + asdl_stmt_seq* block_var; + while ( + (block_var = block_rule(p)) // block + ) + { + _res = block_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_217[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop1_218: except_star_block +static asdl_seq * +_loop1_218_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + p->error_indicator = 1; + PyErr_NoMemory(); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // except_star_block + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); + excepthandler_ty except_star_block_var; + while ( + (except_star_block_var = except_star_block_rule(p)) // except_star_block + ) + { + _res = except_star_block_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_218[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _tmp_219: expression ['as' NAME] +static void * +_tmp_219_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38864,22 +38987,22 @@ _tmp_217_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c> _tmp_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_269_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_271_rule(p), !p->error_indicator) // ['as' NAME] ) { - D(fprintf(stderr, "%*c+ _tmp_217[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c+ _tmp_219[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_217[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_219[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); } _res = NULL; @@ -38888,9 +39011,9 @@ _tmp_217_rule(Parser *p) return _res; } -// _tmp_218: 'as' NAME +// _tmp_220: 'as' NAME static void * -_tmp_218_rule(Parser *p) +_tmp_220_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38907,7 +39030,7 @@ _tmp_218_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38916,12 +39039,12 @@ _tmp_218_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_218[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_218[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38930,9 +39053,9 @@ _tmp_218_rule(Parser *p) return _res; } -// _tmp_219: 'as' NAME +// _tmp_221: 'as' NAME static void * -_tmp_219_rule(Parser *p) +_tmp_221_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38949,7 +39072,7 @@ _tmp_219_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38958,12 +39081,12 @@ _tmp_219_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_219[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_219[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38972,9 +39095,9 @@ _tmp_219_rule(Parser *p) return _res; } -// _tmp_220: NEWLINE | ':' +// _tmp_222: NEWLINE | ':' static void * -_tmp_220_rule(Parser *p) +_tmp_222_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38991,18 +39114,18 @@ _tmp_220_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } { // ':' @@ -39010,18 +39133,18 @@ _tmp_220_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -39030,9 +39153,9 @@ _tmp_220_rule(Parser *p) return _res; } -// _tmp_221: 'as' NAME +// _tmp_223: 'as' NAME static void * -_tmp_221_rule(Parser *p) +_tmp_223_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39049,7 +39172,7 @@ _tmp_221_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -39058,12 +39181,12 @@ _tmp_221_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -39072,9 +39195,9 @@ _tmp_221_rule(Parser *p) return _res; } -// _tmp_222: 'as' NAME +// _tmp_224: 'as' NAME static void * -_tmp_222_rule(Parser *p) +_tmp_224_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39091,7 +39214,7 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -39100,12 +39223,12 @@ _tmp_222_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -39114,9 +39237,9 @@ _tmp_222_rule(Parser *p) return _res; } -// _tmp_223: positional_patterns ',' +// _tmp_225: positional_patterns ',' static void * -_tmp_223_rule(Parser *p) +_tmp_225_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39133,7 +39256,7 @@ _tmp_223_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; asdl_pattern_seq* positional_patterns_var; if ( @@ -39142,12 +39265,12 @@ _tmp_223_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; @@ -39156,9 +39279,9 @@ _tmp_223_rule(Parser *p) return _res; } -// _tmp_224: '->' expression +// _tmp_226: '->' expression static void * -_tmp_224_rule(Parser *p) +_tmp_226_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39175,7 +39298,7 @@ _tmp_224_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); Token * _literal; expr_ty expression_var; if ( @@ -39184,12 +39307,12 @@ _tmp_224_rule(Parser *p) (expression_var = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = _PyPegen_dummy_name(p, _literal, expression_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); } _res = NULL; @@ -39198,9 +39321,9 @@ _tmp_224_rule(Parser *p) return _res; } -// _tmp_225: '(' arguments? ')' +// _tmp_227: '(' arguments? ')' static void * -_tmp_225_rule(Parser *p) +_tmp_227_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39217,7 +39340,7 @@ _tmp_225_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -39230,12 +39353,12 @@ _tmp_225_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -39244,9 +39367,9 @@ _tmp_225_rule(Parser *p) return _res; } -// _tmp_226: '(' arguments? ')' +// _tmp_228: '(' arguments? ')' static void * -_tmp_226_rule(Parser *p) +_tmp_228_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39263,7 +39386,7 @@ _tmp_226_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -39276,12 +39399,12 @@ _tmp_226_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -39290,9 +39413,9 @@ _tmp_226_rule(Parser *p) return _res; } -// _loop0_228: ',' double_starred_kvpair +// _loop0_230: ',' double_starred_kvpair static asdl_seq * -_loop0_228_rule(Parser *p) +_loop0_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39318,7 +39441,7 @@ _loop0_228_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -39350,7 +39473,7 @@ _loop0_228_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_228[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39367,9 +39490,9 @@ _loop0_228_rule(Parser *p) return _seq; } -// _gather_227: double_starred_kvpair _loop0_228 +// _gather_229: double_starred_kvpair _loop0_230 static asdl_seq * -_gather_227_rule(Parser *p) +_gather_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39381,27 +39504,27 @@ _gather_227_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_228 + { // double_starred_kvpair _loop0_230 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_228")); + D(fprintf(stderr, "%*c> _gather_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_230")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_228_rule(p)) // _loop0_228 + (seq = _loop0_230_rule(p)) // _loop0_230 ) { - D(fprintf(stderr, "%*c+ _gather_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_228")); + D(fprintf(stderr, "%*c+ _gather_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_230")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_227[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_228")); + D(fprintf(stderr, "%*c%s _gather_229[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_230")); } _res = NULL; done: @@ -39409,9 +39532,9 @@ _gather_227_rule(Parser *p) return _res; } -// _tmp_229: '}' | ',' +// _tmp_231: '}' | ',' static void * -_tmp_229_rule(Parser *p) +_tmp_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39428,18 +39551,18 @@ _tmp_229_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39447,18 +39570,18 @@ _tmp_229_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39467,9 +39590,9 @@ _tmp_229_rule(Parser *p) return _res; } -// _tmp_230: '}' | ',' +// _tmp_232: '}' | ',' static void * -_tmp_230_rule(Parser *p) +_tmp_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39486,18 +39609,18 @@ _tmp_230_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39505,18 +39628,18 @@ _tmp_230_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39525,9 +39648,9 @@ _tmp_230_rule(Parser *p) return _res; } -// _tmp_231: yield_expr | star_expressions +// _tmp_233: yield_expr | star_expressions static void * -_tmp_231_rule(Parser *p) +_tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39544,18 +39667,18 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39563,18 +39686,18 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39583,9 +39706,9 @@ _tmp_231_rule(Parser *p) return _res; } -// _tmp_232: yield_expr | star_expressions +// _tmp_234: yield_expr | star_expressions static void * -_tmp_232_rule(Parser *p) +_tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39602,18 +39725,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39621,18 +39744,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39641,9 +39764,9 @@ _tmp_232_rule(Parser *p) return _res; } -// _tmp_233: '=' | '!' | ':' | '}' +// _tmp_235: '=' | '!' | ':' | '}' static void * -_tmp_233_rule(Parser *p) +_tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39660,18 +39783,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // '!' @@ -39679,18 +39802,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39698,18 +39821,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39717,18 +39840,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39737,9 +39860,9 @@ _tmp_233_rule(Parser *p) return _res; } -// _tmp_234: yield_expr | star_expressions +// _tmp_236: yield_expr | star_expressions static void * -_tmp_234_rule(Parser *p) +_tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39756,18 +39879,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39775,18 +39898,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39795,9 +39918,9 @@ _tmp_234_rule(Parser *p) return _res; } -// _tmp_235: '!' | ':' | '}' +// _tmp_237: '!' | ':' | '}' static void * -_tmp_235_rule(Parser *p) +_tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39814,18 +39937,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39833,18 +39956,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39852,18 +39975,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39872,9 +39995,9 @@ _tmp_235_rule(Parser *p) return _res; } -// _tmp_236: yield_expr | star_expressions +// _tmp_238: yield_expr | star_expressions static void * -_tmp_236_rule(Parser *p) +_tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39891,18 +40014,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39910,18 +40033,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39930,9 +40053,9 @@ _tmp_236_rule(Parser *p) return _res; } -// _tmp_237: yield_expr | star_expressions +// _tmp_239: yield_expr | star_expressions static void * -_tmp_237_rule(Parser *p) +_tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -39949,18 +40072,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39968,18 +40091,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39988,9 +40111,9 @@ _tmp_237_rule(Parser *p) return _res; } -// _tmp_238: '!' NAME +// _tmp_240: '!' NAME static void * -_tmp_238_rule(Parser *p) +_tmp_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40007,7 +40130,7 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -40016,12 +40139,12 @@ _tmp_238_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -40030,9 +40153,9 @@ _tmp_238_rule(Parser *p) return _res; } -// _tmp_239: ':' | '}' +// _tmp_241: ':' | '}' static void * -_tmp_239_rule(Parser *p) +_tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40049,18 +40172,18 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -40068,18 +40191,18 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -40088,9 +40211,9 @@ _tmp_239_rule(Parser *p) return _res; } -// _tmp_240: yield_expr | star_expressions +// _tmp_242: yield_expr | star_expressions static void * -_tmp_240_rule(Parser *p) +_tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40107,18 +40230,18 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -40126,18 +40249,18 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -40146,9 +40269,9 @@ _tmp_240_rule(Parser *p) return _res; } -// _tmp_241: '!' NAME +// _tmp_243: '!' NAME static void * -_tmp_241_rule(Parser *p) +_tmp_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40165,7 +40288,7 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -40174,12 +40297,12 @@ _tmp_241_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -40188,9 +40311,9 @@ _tmp_241_rule(Parser *p) return _res; } -// _loop0_242: fstring_format_spec +// _loop0_244: fstring_format_spec static asdl_seq * -_loop0_242_rule(Parser *p) +_loop0_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40216,7 +40339,7 @@ _loop0_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); expr_ty fstring_format_spec_var; while ( (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec @@ -40239,7 +40362,7 @@ _loop0_242_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -40256,9 +40379,9 @@ _loop0_242_rule(Parser *p) return _seq; } -// _tmp_243: yield_expr | star_expressions +// _tmp_245: yield_expr | star_expressions static void * -_tmp_243_rule(Parser *p) +_tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40275,18 +40398,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -40294,18 +40417,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -40314,9 +40437,9 @@ _tmp_243_rule(Parser *p) return _res; } -// _tmp_244: '!' NAME +// _tmp_246: '!' NAME static void * -_tmp_244_rule(Parser *p) +_tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40333,7 +40456,7 @@ _tmp_244_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -40342,12 +40465,12 @@ _tmp_244_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -40356,9 +40479,9 @@ _tmp_244_rule(Parser *p) return _res; } -// _tmp_245: ':' | '}' +// _tmp_247: ':' | '}' static void * -_tmp_245_rule(Parser *p) +_tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40375,18 +40498,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -40394,18 +40517,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -40414,9 +40537,9 @@ _tmp_245_rule(Parser *p) return _res; } -// _tmp_246: star_targets '=' +// _tmp_248: star_targets '=' static void * -_tmp_246_rule(Parser *p) +_tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40433,7 +40556,7 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -40442,7 +40565,7 @@ _tmp_246_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40452,7 +40575,7 @@ _tmp_246_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40461,9 +40584,9 @@ _tmp_246_rule(Parser *p) return _res; } -// _tmp_247: '.' | '...' +// _tmp_249: '.' | '...' static void * -_tmp_247_rule(Parser *p) +_tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40480,18 +40603,18 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40499,18 +40622,18 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40519,9 +40642,9 @@ _tmp_247_rule(Parser *p) return _res; } -// _tmp_248: '.' | '...' +// _tmp_250: '.' | '...' static void * -_tmp_248_rule(Parser *p) +_tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40538,18 +40661,18 @@ _tmp_248_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40557,18 +40680,18 @@ _tmp_248_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40577,9 +40700,9 @@ _tmp_248_rule(Parser *p) return _res; } -// _tmp_249: '@' named_expression NEWLINE +// _tmp_251: '@' named_expression NEWLINE static void * -_tmp_249_rule(Parser *p) +_tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40596,7 +40719,7 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -40608,7 +40731,7 @@ _tmp_249_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40618,7 +40741,7 @@ _tmp_249_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -40627,9 +40750,9 @@ _tmp_249_rule(Parser *p) return _res; } -// _tmp_250: ',' expression +// _tmp_252: ',' expression static void * -_tmp_250_rule(Parser *p) +_tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40646,7 +40769,7 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty c; if ( @@ -40655,7 +40778,7 @@ _tmp_250_rule(Parser *p) (c = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40665,7 +40788,7 @@ _tmp_250_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -40674,9 +40797,9 @@ _tmp_250_rule(Parser *p) return _res; } -// _tmp_251: ',' star_expression +// _tmp_253: ',' star_expression static void * -_tmp_251_rule(Parser *p) +_tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40693,7 +40816,7 @@ _tmp_251_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -40702,7 +40825,7 @@ _tmp_251_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40712,7 +40835,7 @@ _tmp_251_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -40721,9 +40844,9 @@ _tmp_251_rule(Parser *p) return _res; } -// _tmp_252: 'or' conjunction +// _tmp_254: 'or' conjunction static void * -_tmp_252_rule(Parser *p) +_tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40740,7 +40863,7 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -40749,7 +40872,7 @@ _tmp_252_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40759,7 +40882,7 @@ _tmp_252_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -40768,9 +40891,9 @@ _tmp_252_rule(Parser *p) return _res; } -// _tmp_253: 'and' inversion +// _tmp_255: 'and' inversion static void * -_tmp_253_rule(Parser *p) +_tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40787,7 +40910,7 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -40796,7 +40919,7 @@ _tmp_253_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40806,7 +40929,7 @@ _tmp_253_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -40815,9 +40938,9 @@ _tmp_253_rule(Parser *p) return _res; } -// _tmp_254: slice | starred_expression +// _tmp_256: slice | starred_expression static void * -_tmp_254_rule(Parser *p) +_tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40834,18 +40957,18 @@ _tmp_254_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -40853,18 +40976,18 @@ _tmp_254_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -40873,9 +40996,9 @@ _tmp_254_rule(Parser *p) return _res; } -// _tmp_255: fstring | string +// _tmp_257: fstring | string static void * -_tmp_255_rule(Parser *p) +_tmp_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40892,18 +41015,18 @@ _tmp_255_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); expr_ty fstring_var; if ( (fstring_var = fstring_rule(p)) // fstring ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); _res = fstring_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); } { // string @@ -40911,18 +41034,18 @@ _tmp_255_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); expr_ty string_var; if ( (string_var = string_rule(p)) // string ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); } _res = NULL; @@ -40931,9 +41054,9 @@ _tmp_255_rule(Parser *p) return _res; } -// _tmp_256: 'if' disjunction +// _tmp_258: 'if' disjunction static void * -_tmp_256_rule(Parser *p) +_tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40950,7 +41073,7 @@ _tmp_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -40959,7 +41082,7 @@ _tmp_256_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40969,7 +41092,7 @@ _tmp_256_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40978,9 +41101,9 @@ _tmp_256_rule(Parser *p) return _res; } -// _tmp_257: 'if' disjunction +// _tmp_259: 'if' disjunction static void * -_tmp_257_rule(Parser *p) +_tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -40997,7 +41120,7 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -41006,7 +41129,7 @@ _tmp_257_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41016,7 +41139,7 @@ _tmp_257_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -41025,9 +41148,9 @@ _tmp_257_rule(Parser *p) return _res; } -// _tmp_258: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_260: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_258_rule(Parser *p) +_tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41044,18 +41167,18 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -41063,20 +41186,20 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_270_var; + D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_272_var; if ( - (_tmp_270_var = _tmp_270_rule(p)) // assignment_expression | expression !':=' + (_tmp_272_var = _tmp_272_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_270_var; + D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_272_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -41085,9 +41208,9 @@ _tmp_258_rule(Parser *p) return _res; } -// _tmp_259: ',' star_target +// _tmp_261: ',' star_target static void * -_tmp_259_rule(Parser *p) +_tmp_261_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41104,7 +41227,7 @@ _tmp_259_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -41113,7 +41236,7 @@ _tmp_259_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41123,7 +41246,7 @@ _tmp_259_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -41132,9 +41255,9 @@ _tmp_259_rule(Parser *p) return _res; } -// _tmp_260: ',' star_target +// _tmp_262: ',' star_target static void * -_tmp_260_rule(Parser *p) +_tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41151,7 +41274,7 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -41160,7 +41283,7 @@ _tmp_260_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41170,7 +41293,7 @@ _tmp_260_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -41179,9 +41302,9 @@ _tmp_260_rule(Parser *p) return _res; } -// _tmp_261: star_targets '=' +// _tmp_263: star_targets '=' static void * -_tmp_261_rule(Parser *p) +_tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41198,7 +41321,7 @@ _tmp_261_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41207,12 +41330,12 @@ _tmp_261_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41221,9 +41344,9 @@ _tmp_261_rule(Parser *p) return _res; } -// _tmp_262: star_targets '=' +// _tmp_264: star_targets '=' static void * -_tmp_262_rule(Parser *p) +_tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41240,7 +41363,7 @@ _tmp_262_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41249,12 +41372,12 @@ _tmp_262_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41263,9 +41386,9 @@ _tmp_262_rule(Parser *p) return _res; } -// _tmp_263: ')' | '**' +// _tmp_265: ')' | '**' static void * -_tmp_263_rule(Parser *p) +_tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41282,18 +41405,18 @@ _tmp_263_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -41301,18 +41424,18 @@ _tmp_263_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41321,9 +41444,9 @@ _tmp_263_rule(Parser *p) return _res; } -// _tmp_264: ':' | '**' +// _tmp_266: ':' | '**' static void * -_tmp_264_rule(Parser *p) +_tmp_266_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41340,18 +41463,18 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -41359,18 +41482,18 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41379,9 +41502,9 @@ _tmp_264_rule(Parser *p) return _res; } -// _tmp_265: expression ['as' star_target] +// _tmp_267: expression ['as' star_target] static void * -_tmp_265_rule(Parser *p) +_tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41398,22 +41521,22 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_271_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_273_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41422,9 +41545,9 @@ _tmp_265_rule(Parser *p) return _res; } -// _tmp_266: expressions ['as' star_target] +// _tmp_268: expressions ['as' star_target] static void * -_tmp_266_rule(Parser *p) +_tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41441,22 +41564,22 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_272_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_274_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41465,9 +41588,9 @@ _tmp_266_rule(Parser *p) return _res; } -// _tmp_267: expression ['as' star_target] +// _tmp_269: expression ['as' star_target] static void * -_tmp_267_rule(Parser *p) +_tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41484,22 +41607,22 @@ _tmp_267_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_273_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_275_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41508,9 +41631,9 @@ _tmp_267_rule(Parser *p) return _res; } -// _tmp_268: expressions ['as' star_target] +// _tmp_270: expressions ['as' star_target] static void * -_tmp_268_rule(Parser *p) +_tmp_270_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41527,22 +41650,22 @@ _tmp_268_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_274_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_276_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41551,9 +41674,9 @@ _tmp_268_rule(Parser *p) return _res; } -// _tmp_269: 'as' NAME +// _tmp_271: 'as' NAME static void * -_tmp_269_rule(Parser *p) +_tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41570,7 +41693,7 @@ _tmp_269_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -41579,12 +41702,12 @@ _tmp_269_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -41593,9 +41716,9 @@ _tmp_269_rule(Parser *p) return _res; } -// _tmp_270: assignment_expression | expression !':=' +// _tmp_272: assignment_expression | expression !':=' static void * -_tmp_270_rule(Parser *p) +_tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41612,18 +41735,18 @@ _tmp_270_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -41631,7 +41754,7 @@ _tmp_270_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -41639,12 +41762,12 @@ _tmp_270_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -41653,9 +41776,9 @@ _tmp_270_rule(Parser *p) return _res; } -// _tmp_271: 'as' star_target +// _tmp_273: 'as' star_target static void * -_tmp_271_rule(Parser *p) +_tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41672,7 +41795,7 @@ _tmp_271_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41681,12 +41804,12 @@ _tmp_271_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41695,9 +41818,9 @@ _tmp_271_rule(Parser *p) return _res; } -// _tmp_272: 'as' star_target +// _tmp_274: 'as' star_target static void * -_tmp_272_rule(Parser *p) +_tmp_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41714,7 +41837,7 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41723,12 +41846,12 @@ _tmp_272_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41737,9 +41860,9 @@ _tmp_272_rule(Parser *p) return _res; } -// _tmp_273: 'as' star_target +// _tmp_275: 'as' star_target static void * -_tmp_273_rule(Parser *p) +_tmp_275_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41756,7 +41879,7 @@ _tmp_273_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41765,12 +41888,12 @@ _tmp_273_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41779,9 +41902,9 @@ _tmp_273_rule(Parser *p) return _res; } -// _tmp_274: 'as' star_target +// _tmp_276: 'as' star_target static void * -_tmp_274_rule(Parser *p) +_tmp_276_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -41798,7 +41921,7 @@ _tmp_274_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41807,12 +41930,12 @@ _tmp_274_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index d4ce33850f7c58..20459e89463494 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -12,6 +12,11 @@ static int warn_invalid_escape_sequence(Parser *p, const char *first_invalid_escape, Token *t) { unsigned char c = *first_invalid_escape; + if ((t->type == FSTRING_MIDDLE || t->type == FSTRING_END) && (c == '{' || c == '}')) { // in this case the tokenizer has already emitted a warning, + // see tokenizer.c:warn_invalid_escape_sequence + return 0; + } + int octal = ('4' <= c && c <= '7'); PyObject *msg = octal @@ -31,7 +36,7 @@ warn_invalid_escape_sequence(Parser *p, const char *first_invalid_escape, Token if (PyErr_WarnExplicitObject(category, msg, p->tok->filename, t->lineno, NULL, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { - /* Replace the DeprecationWarning exception with a SyntaxError + /* Replace the Syntax/DeprecationWarning exception with a SyntaxError to get a more accurate error report */ PyErr_Clear(); diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 57c98fe5fcaa40..115b497cc24090 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1039,9 +1039,6 @@ tok_readline_raw(struct tok_state *tok) if (line == NULL) { return 1; } - if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { - return 0; - } if (tok->fp_interactive && tok_concatenate_interactive_new_line(tok, line) == -1) { return 0; @@ -1106,11 +1103,7 @@ tok_readline_string(struct tok_state* tok) { tok->inp += buflen; *tok->inp = '\0'; - if (tok->start == NULL) { - tok->buf = tok->cur; - } tok->line_start = tok->cur; - Py_DECREF(line); return 1; error: @@ -1274,6 +1267,10 @@ tok_underflow_file(struct tok_state *tok) { tok->implicit_newline = 1; } + if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { + return 0; + } + ADVANCE_LINENO(); if (tok->decoding_state != STATE_NORMAL) { if (tok->lineno > 2) { @@ -1318,6 +1315,10 @@ tok_underflow_readline(struct tok_state* tok) { tok->implicit_newline = 1; } + if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { + return 0; + } + ADVANCE_LINENO(); /* The default encoding is UTF-8, so make sure we don't have any non-UTF-8 sequences in it. */ @@ -1558,12 +1559,12 @@ warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_cha return -1; } - if (PyErr_WarnExplicitObject(PyExc_DeprecationWarning, msg, tok->filename, + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, tok->filename, tok->lineno, NULL, NULL) < 0) { Py_DECREF(msg); - if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { - /* Replace the DeprecationWarning exception with a SyntaxError + if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { + /* Replace the SyntaxWarning exception with a SyntaxError to get a more accurate error report */ PyErr_Clear(); return syntaxerror(tok, "invalid escape sequence '\\%c'", (char) first_invalid_escape_char); diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 16e919a8931edd..cb44845c1d306e 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -68,7 +68,7 @@ typedef struct _tokenizer_mode { struct tok_state { /* Input state; buf <= cur <= inp <= end */ /* NB an entire line is held in the buffer */ - char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */ + char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ int fp_interactive; /* If the file descriptor is interactive */ diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 84bce59e271471..5db9ade3af4547 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1902,12 +1902,6 @@ init_types(struct ast_state *state) if (!state->type_param_type) return 0; if (!add_attributes(state, state->type_param_type, type_param_attributes, 4)) return 0; - if (PyObject_SetAttr(state->type_param_type, state->end_lineno, Py_None) == - -1) - return 0; - if (PyObject_SetAttr(state->type_param_type, state->end_col_offset, - Py_None) == -1) - return 0; state->TypeVar_type = make_type(state, "TypeVar", state->type_param_type, TypeVar_fields, 2, "TypeVar(identifier name, expr? bound)"); @@ -10834,6 +10828,7 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* Py_CLEAR(tmp); } *out = _PyAST_comprehension(target, iter, ifs, is_async, arena); + if (*out == NULL) goto failed; return 0; failed: Py_XDECREF(tmp); @@ -11258,6 +11253,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, } *out = _PyAST_arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults, arena); + if (*out == NULL) goto failed; return 0; failed: Py_XDECREF(tmp); @@ -11397,6 +11393,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) } *out = _PyAST_arg(arg, annotation, type_comment, lineno, col_offset, end_lineno, end_col_offset, arena); + if (*out == NULL) goto failed; return 0; failed: Py_XDECREF(tmp); @@ -11519,6 +11516,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, } *out = _PyAST_keyword(arg, value, lineno, col_offset, end_lineno, end_col_offset, arena); + if (*out == NULL) goto failed; return 0; failed: Py_XDECREF(tmp); @@ -11641,6 +11639,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* } *out = _PyAST_alias(name, asname, lineno, col_offset, end_lineno, end_col_offset, arena); + if (*out == NULL) goto failed; return 0; failed: Py_XDECREF(tmp); @@ -11690,6 +11689,7 @@ obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, Py_CLEAR(tmp); } *out = _PyAST_withitem(context_expr, optional_vars, arena); + if (*out == NULL) goto failed; return 0; failed: Py_XDECREF(tmp); @@ -11778,6 +11778,7 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, Py_CLEAR(tmp); } *out = _PyAST_match_case(pattern, guard, body, arena); + if (*out == NULL) goto failed; return 0; failed: Py_XDECREF(tmp); @@ -12493,9 +12494,9 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_PyObject_LookupAttr(obj, state->end_lineno, &tmp) < 0) { return 1; } - if (tmp == NULL || tmp == Py_None) { - Py_CLEAR(tmp); - end_lineno = lineno; + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"end_lineno\" missing from type_param"); + return 1; } else { int res; @@ -12510,9 +12511,9 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_PyObject_LookupAttr(obj, state->end_col_offset, &tmp) < 0) { return 1; } - if (tmp == NULL || tmp == Py_None) { - Py_CLEAR(tmp); - end_col_offset = col_offset; + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"end_col_offset\" missing from type_param"); + return 1; } else { int res; diff --git a/Python/_warnings.c b/Python/_warnings.c index dec658680241ed..1f91edbf5cb5dc 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1301,25 +1301,29 @@ PyErr_WarnExplicit(PyObject *category, const char *text, const char *module_str, PyObject *registry) { PyObject *message = PyUnicode_FromString(text); + if (message == NULL) { + return -1; + } PyObject *filename = PyUnicode_DecodeFSDefault(filename_str); + if (filename == NULL) { + Py_DECREF(message); + return -1; + } PyObject *module = NULL; - int ret = -1; - - if (message == NULL || filename == NULL) - goto exit; if (module_str != NULL) { module = PyUnicode_FromString(module_str); - if (module == NULL) - goto exit; + if (module == NULL) { + Py_DECREF(filename); + Py_DECREF(message); + return -1; + } } - ret = PyErr_WarnExplicitObject(category, message, filename, lineno, - module, registry); - - exit: - Py_XDECREF(message); + int ret = PyErr_WarnExplicitObject(category, message, filename, lineno, + module, registry); Py_XDECREF(module); - Py_XDECREF(filename); + Py_DECREF(filename); + Py_DECREF(message); return ret; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ddddc03ca316e0..7f366b43599ae5 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -907,7 +907,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) /*[clinic end generated code: output=0a0824aa70093116 input=11ee718a8640e527]*/ { - PyObject *result, *source_copy; + PyObject *result = NULL, *source_copy; const char *str; if (locals != Py_None && !PyMapping_Check(locals)) { @@ -923,19 +923,25 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, if (globals == Py_None) { globals = PyEval_GetGlobals(); if (locals == Py_None) { - locals = PyEval_GetLocals(); + locals = _PyEval_GetFrameLocals(); if (locals == NULL) return NULL; } + else { + Py_INCREF(locals); + } } else if (locals == Py_None) - locals = globals; + locals = Py_NewRef(globals); + else { + Py_INCREF(locals); + } if (globals == NULL || locals == NULL) { PyErr_SetString(PyExc_TypeError, "eval must be given globals and locals " "when called without a frame"); - return NULL; + goto error; } int r = PyDict_Contains(globals, &_Py_ID(__builtins__)); @@ -943,34 +949,38 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, r = PyDict_SetItem(globals, &_Py_ID(__builtins__), PyEval_GetBuiltins()); } if (r < 0) { - return NULL; + goto error; } if (PyCode_Check(source)) { if (PySys_Audit("exec", "O", source) < 0) { - return NULL; + goto error; } if (PyCode_GetNumFree((PyCodeObject *)source) > 0) { PyErr_SetString(PyExc_TypeError, "code object passed to eval() may not contain free variables"); - return NULL; + goto error; } - return PyEval_EvalCode(source, globals, locals); + result = PyEval_EvalCode(source, globals, locals); } + else { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + cf.cf_flags = PyCF_SOURCE_IS_UTF8; + str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy); + if (str == NULL) + goto error; - PyCompilerFlags cf = _PyCompilerFlags_INIT; - cf.cf_flags = PyCF_SOURCE_IS_UTF8; - str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy); - if (str == NULL) - return NULL; + while (*str == ' ' || *str == '\t') + str++; - while (*str == ' ' || *str == '\t') - str++; + (void)PyEval_MergeCompilerFlags(&cf); + result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf); + Py_XDECREF(source_copy); + } - (void)PyEval_MergeCompilerFlags(&cf); - result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf); - Py_XDECREF(source_copy); + error: + Py_XDECREF(locals); return result; } @@ -1005,36 +1015,43 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, if (globals == Py_None) { globals = PyEval_GetGlobals(); if (locals == Py_None) { - locals = PyEval_GetLocals(); + locals = _PyEval_GetFrameLocals(); if (locals == NULL) return NULL; } + else { + Py_INCREF(locals); + } if (!globals || !locals) { PyErr_SetString(PyExc_SystemError, "globals and locals cannot be NULL"); return NULL; } } - else if (locals == Py_None) - locals = globals; + else if (locals == Py_None) { + locals = Py_NewRef(globals); + } + else { + Py_INCREF(locals); + } if (!PyDict_Check(globals)) { PyErr_Format(PyExc_TypeError, "exec() globals must be a dict, not %.100s", Py_TYPE(globals)->tp_name); - return NULL; + goto error; } if (!PyMapping_Check(locals)) { PyErr_Format(PyExc_TypeError, "locals must be a mapping or None, not %.100s", Py_TYPE(locals)->tp_name); - return NULL; + goto error; } int r = PyDict_Contains(globals, &_Py_ID(__builtins__)); if (r == 0) { r = PyDict_SetItem(globals, &_Py_ID(__builtins__), PyEval_GetBuiltins()); } if (r < 0) { - return NULL; + goto error; } if (closure == Py_None) { @@ -1047,7 +1064,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, if (closure) { PyErr_SetString(PyExc_TypeError, "cannot use a closure with this code object"); - return NULL; + goto error; } } else { int closure_is_ok = @@ -1067,12 +1084,12 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyErr_Format(PyExc_TypeError, "code object requires a closure of exactly length %zd", num_free); - return NULL; + goto error; } } if (PySys_Audit("exec", "O", source) < 0) { - return NULL; + goto error; } if (!closure) { @@ -1099,7 +1116,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, "string, bytes or code", &cf, &source_copy); if (str == NULL) - return NULL; + goto error; if (PyEval_MergeCompilerFlags(&cf)) v = PyRun_StringFlags(str, Py_file_input, globals, locals, &cf); @@ -1108,9 +1125,14 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, Py_XDECREF(source_copy); } if (v == NULL) - return NULL; + goto error; + Py_DECREF(locals); Py_DECREF(v); Py_RETURN_NONE; + + error: + Py_XDECREF(locals); + return NULL; } @@ -1720,10 +1742,7 @@ static PyObject * builtin_locals_impl(PyObject *module) /*[clinic end generated code: output=b46c94015ce11448 input=7874018d478d5c4b]*/ { - PyObject *d; - - d = PyEval_GetLocals(); - return Py_XNewRef(d); + return _PyEval_GetFrameLocals(); } @@ -2164,17 +2183,29 @@ builtin_input_impl(PyObject *module, PyObject *prompt) /* stdin is a text stream, so it must have an encoding. */ stdin_encoding = PyObject_GetAttr(fin, &_Py_ID(encoding)); + if (stdin_encoding == NULL) { + tty = 0; + goto _readline_errors; + } stdin_errors = PyObject_GetAttr(fin, &_Py_ID(errors)); - if (!stdin_encoding || !stdin_errors || - !PyUnicode_Check(stdin_encoding) || - !PyUnicode_Check(stdin_errors)) { + if (stdin_errors == NULL) { + tty = 0; + goto _readline_errors; + } + if (!PyUnicode_Check(stdin_encoding) || + !PyUnicode_Check(stdin_errors)) + { tty = 0; goto _readline_errors; } stdin_encoding_str = PyUnicode_AsUTF8(stdin_encoding); + if (stdin_encoding_str == NULL) { + goto _readline_errors; + } stdin_errors_str = PyUnicode_AsUTF8(stdin_errors); - if (!stdin_encoding_str || !stdin_errors_str) + if (stdin_errors_str == NULL) { goto _readline_errors; + } tmp = PyObject_CallMethodNoArgs(fout, &_Py_ID(flush)); if (tmp == NULL) PyErr_Clear(); @@ -2185,17 +2216,29 @@ builtin_input_impl(PyObject *module, PyObject *prompt) const char *stdout_encoding_str, *stdout_errors_str; PyObject *stringpo; stdout_encoding = PyObject_GetAttr(fout, &_Py_ID(encoding)); + if (stdout_encoding == NULL) { + tty = 0; + goto _readline_errors; + } stdout_errors = PyObject_GetAttr(fout, &_Py_ID(errors)); - if (!stdout_encoding || !stdout_errors || - !PyUnicode_Check(stdout_encoding) || - !PyUnicode_Check(stdout_errors)) { + if (stdout_errors == NULL) { + tty = 0; + goto _readline_errors; + } + if (!PyUnicode_Check(stdout_encoding) || + !PyUnicode_Check(stdout_errors)) + { tty = 0; goto _readline_errors; } stdout_encoding_str = PyUnicode_AsUTF8(stdout_encoding); + if (stdout_encoding_str == NULL) { + goto _readline_errors; + } stdout_errors_str = PyUnicode_AsUTF8(stdout_errors); - if (!stdout_encoding_str || !stdout_errors_str) + if (stdout_errors_str == NULL) { goto _readline_errors; + } stringpo = PyObject_Str(prompt); if (stringpo == NULL) goto _readline_errors; @@ -2419,7 +2462,7 @@ builtin_vars_impl(PyObject *module, PyObject *object) PyObject *d; if (object == NULL) { - d = Py_XNewRef(PyEval_GetLocals()); + d = _PyEval_GetFrameLocals(); } else { if (_PyObject_LookupAttr(object, &_Py_ID(__dict__), &d) == 0) { diff --git a/Python/ceval.c b/Python/ceval.c index df997e1ed283d7..c883a903e964f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -758,6 +758,61 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int * We need to do reasonably frequently, but not too frequently. * All loops should include a check of the eval breaker. * We also check on return from any builtin function. + * + * ## More Details ### + * + * The eval loop (this function) normally executes the instructions + * of a code object sequentially. However, the runtime supports a + * number of out-of-band execution scenarios that may pause that + * sequential execution long enough to do that out-of-band work + * in the current thread using the current PyThreadState. + * + * The scenarios include: + * + * - cyclic garbage collection + * - GIL drop requests + * - "async" exceptions + * - "pending calls" (some only in the main thread) + * - signal handling (only in the main thread) + * + * When the need for one of the above is detected, the eval loop + * pauses long enough to handle the detected case. Then, if doing + * so didn't trigger an exception, the eval loop resumes executing + * the sequential instructions. + * + * To make this work, the eval loop periodically checks if any + * of the above needs to happen. The individual checks can be + * expensive if computed each time, so a while back we switched + * to using pre-computed, per-interpreter variables for the checks, + * and later consolidated that to a single "eval breaker" variable + * (now a PyInterpreterState field). + * + * For the longest time, the eval breaker check would happen + * frequently, every 5 or so times through the loop, regardless + * of what instruction ran last or what would run next. Then, in + * early 2021 (gh-18334, commit 4958f5d), we switched to checking + * the eval breaker less frequently, by hard-coding the check to + * specific places in the eval loop (e.g. certain instructions). + * The intent then was to check after returning from calls + * and on the back edges of loops. + * + * In addition to being more efficient, that approach keeps + * the eval loop from running arbitrary code between instructions + * that don't handle that well. (See gh-74174.) + * + * Currently, the eval breaker check happens here at the + * "handle_eval_breaker" label. Some instructions come here + * explicitly (goto) and some indirectly. Notably, the check + * happens on back edges in the control flow graph, which + * pretty much applies to all loops and most calls. + * (See bytecodes.c for exact information.) + * + * One consequence of this approach is that it might not be obvious + * how to force any specific thread to pick up the eval breaker, + * or for any specific thread to not pick it up. Mostly this + * involves judicious uses of locks and careful ordering of code, + * while avoiding code that might trigger the eval breaker + * until so desired. */ if (_Py_HandlePending(tstate) != 0) { goto error; @@ -2257,6 +2312,19 @@ PyEval_GetLocals(void) return locals; } +PyObject * +_PyEval_GetFrameLocals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; + } + + return _PyFrame_GetLocals(current_frame, 1); +} + PyObject * PyEval_GetGlobals(void) { diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 764278ac130dfa..aacf2b5c2e2c4f 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -68,8 +68,9 @@ COMPUTE_EVAL_BREAKER(PyInterpreterState *interp, _Py_atomic_load_relaxed_int32(&ceval2->gil_drop_request) | (_Py_atomic_load_relaxed_int32(&ceval->signals_pending) && _Py_ThreadCanHandleSignals(interp)) - | (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do) - && _Py_ThreadCanHandlePendingCalls()) + | (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do)) + | (_Py_IsMainThread() && _Py_IsMainInterpreter(interp) + &&_Py_atomic_load_relaxed_int32(&ceval->pending_mainthread.calls_to_do)) | ceval2->pending.async_exc | _Py_atomic_load_relaxed_int32(&ceval2->gc_scheduled)); } @@ -95,11 +96,11 @@ RESET_GIL_DROP_REQUEST(PyInterpreterState *interp) static inline void -SIGNAL_PENDING_CALLS(PyInterpreterState *interp) +SIGNAL_PENDING_CALLS(struct _pending_calls *pending, PyInterpreterState *interp) { struct _ceval_runtime_state *ceval = &interp->runtime->ceval; struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 1); + _Py_atomic_store_relaxed(&pending->calls_to_do, 1); COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); } @@ -109,6 +110,9 @@ UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp) { struct _ceval_runtime_state *ceval = &interp->runtime->ceval; struct _ceval_state *ceval2 = &interp->ceval; + if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) { + _Py_atomic_store_relaxed(&ceval->pending_mainthread.calls_to_do, 0); + } _Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 0); COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); } @@ -798,19 +802,31 @@ _push_pending_call(struct _pending_calls *pending, return 0; } -/* Pop one item off the queue while holding the lock. */ -static void -_pop_pending_call(struct _pending_calls *pending, - int (**func)(void *), void **arg) +static int +_next_pending_call(struct _pending_calls *pending, + int (**func)(void *), void **arg) { int i = pending->first; if (i == pending->last) { - return; /* Queue empty */ + /* Queue empty */ + assert(pending->calls[i].func == NULL); + return -1; } - *func = pending->calls[i].func; *arg = pending->calls[i].arg; - pending->first = (i + 1) % NPENDINGCALLS; + return i; +} + +/* Pop one item off the queue while holding the lock. */ +static void +_pop_pending_call(struct _pending_calls *pending, + int (**func)(void *), void **arg) +{ + int i = _next_pending_call(pending, func, arg); + if (i >= 0) { + pending->calls[i] = (struct _pending_call){0}; + pending->first = (i + 1) % NPENDINGCALLS; + } } /* This implementation is thread-safe. It allows @@ -820,9 +836,16 @@ _pop_pending_call(struct _pending_calls *pending, int _PyEval_AddPendingCall(PyInterpreterState *interp, - int (*func)(void *), void *arg) + int (*func)(void *), void *arg, + int mainthreadonly) { + assert(!mainthreadonly || _Py_IsMainInterpreter(interp)); struct _pending_calls *pending = &interp->ceval.pending; + if (mainthreadonly) { + /* The main thread only exists in the main interpreter. */ + assert(_Py_IsMainInterpreter(interp)); + pending = &_PyRuntime.ceval.pending_mainthread; + } /* Ensure that _PyEval_InitState() was called and that _PyEval_FiniState() is not called yet. */ assert(pending->lock != NULL); @@ -832,39 +855,17 @@ _PyEval_AddPendingCall(PyInterpreterState *interp, PyThread_release_lock(pending->lock); /* signal main loop */ - SIGNAL_PENDING_CALLS(interp); + SIGNAL_PENDING_CALLS(pending, interp); return result; } int Py_AddPendingCall(int (*func)(void *), void *arg) { - /* Best-effort to support subinterpreters and calls with the GIL released. - - First attempt _PyThreadState_GET() since it supports subinterpreters. - - If the GIL is released, _PyThreadState_GET() returns NULL . In this - case, use PyGILState_GetThisThreadState() which works even if the GIL - is released. - - Sadly, PyGILState_GetThisThreadState() doesn't support subinterpreters: - see bpo-10915 and bpo-15751. - - Py_AddPendingCall() doesn't require the caller to hold the GIL. */ - PyThreadState *tstate = _PyThreadState_GET(); - if (tstate == NULL) { - tstate = PyGILState_GetThisThreadState(); - } - - PyInterpreterState *interp; - if (tstate != NULL) { - interp = tstate->interp; - } - else { - /* Last resort: use the main interpreter */ - interp = _PyInterpreterState_Main(); - } - return _PyEval_AddPendingCall(interp, func, arg); + /* Legacy users of this API will continue to target the main thread + (of the main interpreter). */ + PyInterpreterState *interp = _PyInterpreterState_Main(); + return _PyEval_AddPendingCall(interp, func, arg, 1); } static int @@ -884,27 +885,24 @@ handle_signals(PyThreadState *tstate) return 0; } -static int -make_pending_calls(PyInterpreterState *interp) +static inline int +maybe_has_pending_calls(PyInterpreterState *interp) { - /* only execute pending calls on main thread */ - if (!_Py_ThreadCanHandlePendingCalls()) { - return 0; + struct _pending_calls *pending = &interp->ceval.pending; + if (_Py_atomic_load_relaxed_int32(&pending->calls_to_do)) { + return 1; } - - /* don't perform recursive pending calls */ - if (interp->ceval.pending.busy) { + if (!_Py_IsMainThread() || !_Py_IsMainInterpreter(interp)) { return 0; } - interp->ceval.pending.busy = 1; - - /* unsignal before starting to call callbacks, so that any callback - added in-between re-signals */ - UNSIGNAL_PENDING_CALLS(interp); - int res = 0; + pending = &_PyRuntime.ceval.pending_mainthread; + return _Py_atomic_load_relaxed_int32(&pending->calls_to_do); +} +static int +_make_pending_calls(struct _pending_calls *pending) +{ /* perform a bounded number of calls, in case of recursion */ - struct _pending_calls *pending = &interp->ceval.pending; for (int i=0; iceval.pending; + struct _pending_calls *pending_main = &_PyRuntime.ceval.pending_mainthread; + + /* Only one thread (per interpreter) may run the pending calls + at once. In the same way, we don't do recursive pending calls. */ + PyThread_acquire_lock(pending->lock, WAIT_LOCK); + if (pending->busy) { + /* A pending call was added after another thread was already + handling the pending calls (and had already "unsignaled"). + Once that thread is done, it may have taken care of all the + pending calls, or there might be some still waiting. + Regardless, this interpreter's pending calls will stay + "signaled" until that first thread has finished. At that + point the next thread to trip the eval breaker will take + care of any remaining pending calls. Until then, though, + all the interpreter's threads will be tripping the eval + breaker every time it's checked. */ + PyThread_release_lock(pending->lock); + return 0; + } + pending->busy = 1; + PyThread_release_lock(pending->lock); + + /* unsignal before starting to call callbacks, so that any callback + added in-between re-signals */ + UNSIGNAL_PENDING_CALLS(interp); + + if (_make_pending_calls(pending) != 0) { + pending->busy = 0; + /* There might not be more calls to make, but we play it safe. */ + SIGNAL_PENDING_CALLS(pending, interp); + return -1; + } - interp->ceval.pending.busy = 0; - return res; + if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) { + if (_make_pending_calls(pending_main) != 0) { + pending->busy = 0; + /* There might not be more calls to make, but we play it safe. */ + SIGNAL_PENDING_CALLS(pending_main, interp); + return -1; + } + } -error: - interp->ceval.pending.busy = 0; - SIGNAL_PENDING_CALLS(interp); - return res; + pending->busy = 0; + return 0; } void @@ -939,12 +979,6 @@ _Py_FinishPendingCalls(PyThreadState *tstate) assert(PyGILState_Check()); assert(is_tstate_valid(tstate)); - struct _pending_calls *pending = &tstate->interp->ceval.pending; - - if (!_Py_atomic_load_relaxed_int32(&(pending->calls_to_do))) { - return; - } - if (make_pending_calls(tstate->interp) < 0) { PyObject *exc = _PyErr_GetRaisedException(tstate); PyErr_BadInternalCall(); @@ -953,6 +987,29 @@ _Py_FinishPendingCalls(PyThreadState *tstate) } } +int +_PyEval_MakePendingCalls(PyThreadState *tstate) +{ + int res; + + if (_Py_IsMainThread() && _Py_IsMainInterpreter(tstate->interp)) { + /* Python signal handler doesn't really queue a callback: + it only signals that a signal was received, + see _PyEval_SignalReceived(). */ + res = handle_signals(tstate); + if (res != 0) { + return res; + } + } + + res = make_pending_calls(tstate->interp); + if (res != 0) { + return res; + } + + return 0; +} + /* Py_MakePendingCalls() is a simple wrapper for the sake of backward-compatibility. */ int @@ -963,19 +1020,11 @@ Py_MakePendingCalls(void) PyThreadState *tstate = _PyThreadState_GET(); assert(is_tstate_valid(tstate)); - /* Python signal handler doesn't really queue a callback: it only signals - that a signal was received, see _PyEval_SignalReceived(). */ - int res = handle_signals(tstate); - if (res != 0) { - return res; - } - - res = make_pending_calls(tstate->interp); - if (res != 0) { - return res; + /* Only execute pending calls on the main thread. */ + if (!_Py_IsMainThread() || !_Py_IsMainInterpreter(tstate->interp)) { + return 0; } - - return 0; + return _PyEval_MakePendingCalls(tstate); } void @@ -1015,7 +1064,7 @@ _Py_HandlePending(PyThreadState *tstate) } /* Pending calls */ - if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->pending.calls_to_do)) { + if (maybe_has_pending_calls(tstate->interp)) { if (make_pending_calls(tstate->interp) != 0) { return -1; } diff --git a/Python/compile.c b/Python/compile.c index f593e957caae97..a8d0016a10786a 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -510,8 +510,10 @@ static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone); static int compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, - PyCompilerFlags flags, int optimize, PyArena *arena) + PyCompilerFlags *flags, int optimize, PyArena *arena) { + PyCompilerFlags local_flags = _PyCompilerFlags_INIT; + c->c_const_cache = PyDict_New(); if (!c->c_const_cache) { return ERROR; @@ -527,10 +529,13 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { return ERROR; } - int merged = c->c_future.ff_features | flags.cf_flags; + if (!flags) { + flags = &local_flags; + } + int merged = c->c_future.ff_features | flags->cf_flags; c->c_future.ff_features = merged; - flags.cf_flags = merged; - c->c_flags = flags; + flags->cf_flags = merged; + c->c_flags = *flags; c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_nestlevel = 0; @@ -555,12 +560,11 @@ static struct compiler* new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int optimize, PyArena *arena) { - PyCompilerFlags flags = pflags ? *pflags : _PyCompilerFlags_INIT; struct compiler *c = PyMem_Calloc(1, sizeof(struct compiler)); if (c == NULL) { return NULL; } - if (compiler_setup(c, mod, filename, flags, optimize, arena) < 0) { + if (compiler_setup(c, mod, filename, pflags, optimize, arena) < 0) { compiler_free(c); return NULL; } diff --git a/Python/import.c b/Python/import.c index 9e1857d5f3e4e6..d10c5ce63a2c8b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -703,10 +703,19 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) _PyRuntime.imports.pkgcontext, and PyModule_Create*() will substitute this (if the name actually matches). */ + +#ifdef HAVE_THREAD_LOCAL +_Py_thread_local const char *pkgcontext = NULL; +# undef PKGCONTEXT +# define PKGCONTEXT pkgcontext +#endif + const char * _PyImport_ResolveNameWithPackageContext(const char *name) { +#ifndef HAVE_THREAD_LOCAL PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK); +#endif if (PKGCONTEXT != NULL) { const char *p = strrchr(PKGCONTEXT, '.'); if (p != NULL && strcmp(name, p+1) == 0) { @@ -714,17 +723,23 @@ _PyImport_ResolveNameWithPackageContext(const char *name) PKGCONTEXT = NULL; } } +#ifndef HAVE_THREAD_LOCAL PyThread_release_lock(EXTENSIONS.mutex); +#endif return name; } const char * _PyImport_SwapPackageContext(const char *newcontext) { +#ifndef HAVE_THREAD_LOCAL PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK); +#endif const char *oldcontext = PKGCONTEXT; PKGCONTEXT = newcontext; +#ifndef HAVE_THREAD_LOCAL PyThread_release_lock(EXTENSIONS.mutex); +#endif return oldcontext; } @@ -2038,6 +2053,7 @@ unmarshal_frozen_code(PyInterpreterState *interp, struct frozen_info *info) PyObject *co = PyMarshal_ReadObjectFromString(info->data, info->size); if (co == NULL) { /* Does not contain executable code. */ + PyErr_Clear(); set_frozen_error(FROZEN_INVALID, info->nameobj); return NULL; } diff --git a/Python/initconfig.c b/Python/initconfig.c index 0d42b7ea082d61..4e5d4bb9876e3b 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -49,7 +49,7 @@ Options (and corresponding environment variables):\n\ .pyc extension; also PYTHONOPTIMIZE=x\n\ -OO : do -O changes and also discard docstrings; add .opt-2 before\n\ .pyc extension\n\ --P : don't prepend a potentially unsafe path to sys.path\n\ +-P : don't prepend a potentially unsafe path to sys.path; also PYTHONSAFEPATH\n\ -q : don't print version and copyright messages on interactive startup\n\ -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ -S : don't imply 'import site' on initialization\n\ @@ -144,7 +144,6 @@ static const char usage_envvars[] = "PYTHONSTARTUP: file executed on interactive startup (no default)\n" "PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" " default module search path. The result is sys.path.\n" -"PYTHONSAFEPATH: don't prepend a potentially unsafe path to sys.path.\n" "PYTHONHOME : alternate directory (or %lc).\n" " The default module search path uses %s.\n" "PYTHONPLATLIBDIR : override sys.platlibdir.\n" @@ -184,6 +183,7 @@ static const char usage_envvars[] = " (-X int_max_str_digits=number)\n" "PYTHONNOUSERSITE : disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" +"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path (-P)\n" "PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n" "PYTHONVERBOSE : trace import statements (-v)\n" "PYTHONWARNINGS=arg : warning control (-W arg)\n"; diff --git a/Python/marshal.c b/Python/marshal.c index 6439503d2c6879..90953cbb728459 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -751,23 +751,28 @@ r_string(Py_ssize_t n, RFILE *p) static int r_byte(RFILE *p) { - int c = EOF; - if (p->ptr != NULL) { - if (p->ptr < p->end) - c = (unsigned char) *p->ptr++; - return c; + if (p->ptr < p->end) { + return (unsigned char) *p->ptr++; + } } - if (!p->readable) { + else if (!p->readable) { assert(p->fp); - c = getc(p->fp); + int c = getc(p->fp); + if (c != EOF) { + return c; + } } else { const char *ptr = r_string(1, p); - if (ptr != NULL) - c = *(const unsigned char *) ptr; + if (ptr != NULL) { + return *(const unsigned char *) ptr; + } + return EOF; } - return c; + PyErr_SetString(PyExc_EOFError, + "EOF read where not expected"); + return EOF; } static int @@ -828,10 +833,11 @@ r_PyLong(RFILE *p) digit d; n = r_long(p); - if (PyErr_Occurred()) - return NULL; if (n == 0) return (PyObject *)_PyLong_New(0); + if (n == -1 && PyErr_Occurred()) { + return NULL; + } if (n < -SIZE32_MAX || n > SIZE32_MAX) { PyErr_SetString(PyExc_ValueError, "bad marshal data (long size out of range)"); @@ -850,10 +856,6 @@ r_PyLong(RFILE *p) d = 0; for (j=0; j < PyLong_MARSHAL_RATIO; j++) { md = r_short(p); - if (PyErr_Occurred()) { - Py_DECREF(ob); - return NULL; - } if (md < 0 || md > PyLong_MARSHAL_BASE) goto bad_digit; d += (digit)md << j*PyLong_MARSHAL_SHIFT; @@ -864,10 +866,6 @@ r_PyLong(RFILE *p) d = 0; for (j=0; j < shorts_in_top_digit; j++) { md = r_short(p); - if (PyErr_Occurred()) { - Py_DECREF(ob); - return NULL; - } if (md < 0 || md > PyLong_MARSHAL_BASE) goto bad_digit; /* topmost marshal digit should be nonzero */ @@ -879,18 +877,17 @@ r_PyLong(RFILE *p) } d += (digit)md << j*PyLong_MARSHAL_SHIFT; } - if (PyErr_Occurred()) { - Py_DECREF(ob); - return NULL; - } + assert(!PyErr_Occurred()); /* top digit should be nonzero, else the resulting PyLong won't be normalized */ ob->long_value.ob_digit[size-1] = d; return (PyObject *)ob; bad_digit: Py_DECREF(ob); - PyErr_SetString(PyExc_ValueError, - "bad marshal data (digit out of range in long)"); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "bad marshal data (digit out of range in long)"); + } return NULL; } @@ -913,8 +910,6 @@ r_float_str(RFILE *p) const char *ptr; n = r_byte(p); if (n == EOF) { - PyErr_SetString(PyExc_EOFError, - "EOF read where object expected"); return -1; } ptr = r_string(n, p); @@ -992,8 +987,10 @@ r_object(RFILE *p) PyObject *retval = NULL; if (code == EOF) { - PyErr_SetString(PyExc_EOFError, - "EOF read where object expected"); + if (PyErr_ExceptionMatches(PyExc_EOFError)) { + PyErr_SetString(PyExc_EOFError, + "EOF read where object expected"); + } return NULL; } @@ -1040,7 +1037,10 @@ r_object(RFILE *p) case TYPE_INT: n = r_long(p); - retval = PyErr_Occurred() ? NULL : PyLong_FromLong(n); + if (n == -1 && PyErr_Occurred()) { + break; + } + retval = PyLong_FromLong(n); R_REF(retval); break; @@ -1106,10 +1106,11 @@ r_object(RFILE *p) { const char *ptr; n = r_long(p); - if (PyErr_Occurred()) - break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (bytes object size out of range)"); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "bad marshal data (bytes object size out of range)"); + } break; } v = PyBytes_FromStringAndSize((char *)NULL, n); @@ -1131,10 +1132,11 @@ r_object(RFILE *p) /* fall through */ case TYPE_ASCII: n = r_long(p); - if (PyErr_Occurred()) - break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "bad marshal data (string size out of range)"); + } break; } goto _read_ascii; @@ -1145,8 +1147,6 @@ r_object(RFILE *p) case TYPE_SHORT_ASCII: n = r_byte(p); if (n == EOF) { - PyErr_SetString(PyExc_EOFError, - "EOF read where object expected"); break; } _read_ascii: @@ -1173,10 +1173,11 @@ r_object(RFILE *p) const char *buffer; n = r_long(p); - if (PyErr_Occurred()) - break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "bad marshal data (string size out of range)"); + } break; } if (n != 0) { @@ -1198,16 +1199,18 @@ r_object(RFILE *p) } case TYPE_SMALL_TUPLE: - n = (unsigned char) r_byte(p); - if (PyErr_Occurred()) + n = r_byte(p); + if (n == EOF) { break; + } goto _read_tuple; case TYPE_TUPLE: n = r_long(p); - if (PyErr_Occurred()) - break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)"); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "bad marshal data (tuple size out of range)"); + } break; } _read_tuple: @@ -1232,10 +1235,11 @@ r_object(RFILE *p) case TYPE_LIST: n = r_long(p); - if (PyErr_Occurred()) - break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)"); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "bad marshal data (list size out of range)"); + } break; } v = PyList_New(n); @@ -1288,10 +1292,11 @@ r_object(RFILE *p) case TYPE_SET: case TYPE_FROZENSET: n = r_long(p); - if (PyErr_Occurred()) - break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)"); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "bad marshal data (set size out of range)"); + } break; } @@ -1368,20 +1373,20 @@ r_object(RFILE *p) /* XXX ignore long->int overflows for now */ argcount = (int)r_long(p); - if (PyErr_Occurred()) + if (argcount == -1 && PyErr_Occurred()) goto code_error; posonlyargcount = (int)r_long(p); - if (PyErr_Occurred()) { + if (posonlyargcount == -1 && PyErr_Occurred()) { goto code_error; } kwonlyargcount = (int)r_long(p); - if (PyErr_Occurred()) + if (kwonlyargcount == -1 && PyErr_Occurred()) goto code_error; stacksize = (int)r_long(p); - if (PyErr_Occurred()) + if (stacksize == -1 && PyErr_Occurred()) goto code_error; flags = (int)r_long(p); - if (PyErr_Occurred()) + if (flags == -1 && PyErr_Occurred()) goto code_error; code = r_object(p); if (code == NULL) @@ -1454,6 +1459,10 @@ r_object(RFILE *p) v = r_ref_insert(v, idx, flag, p); code_error: + if (v == NULL && !PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "NULL object in marshal data for code object"); + } Py_XDECREF(code); Py_XDECREF(consts); Py_XDECREF(names); @@ -1471,9 +1480,10 @@ r_object(RFILE *p) case TYPE_REF: n = r_long(p); if (n < 0 || n >= PyList_GET_SIZE(p->refs)) { - if (n == -1 && PyErr_Occurred()) - break; - PyErr_SetString(PyExc_ValueError, "bad marshal data (invalid reference)"); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "bad marshal data (invalid reference)"); + } break; } v = PyList_GET_ITEM(p->refs, n); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 95b39f478f51f6..a67fa26b372278 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -578,12 +578,14 @@ init_interp_settings(PyInterpreterState *interp, interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS; } + /* We check "gil" in init_interp_create_gil(). */ + return _PyStatus_OK(); } static PyStatus -init_interp_create_gil(PyThreadState *tstate, int own_gil) +init_interp_create_gil(PyThreadState *tstate, int gil) { PyStatus status; @@ -598,6 +600,15 @@ init_interp_create_gil(PyThreadState *tstate, int own_gil) return status; } + int own_gil; + switch (gil) { + case PyInterpreterConfig_DEFAULT_GIL: own_gil = 0; break; + case PyInterpreterConfig_SHARED_GIL: own_gil = 0; break; + case PyInterpreterConfig_OWN_GIL: own_gil = 1; break; + default: + return _PyStatus_ERR("invalid interpreter config 'gil' value"); + } + /* Create the GIL and take it */ status = _PyEval_InitGIL(tstate, own_gil); if (_PyStatus_EXCEPTION(status)) { @@ -633,7 +644,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; // The main interpreter always has its own GIL. - config.own_gil = 1; + config.gil = PyInterpreterConfig_OWN_GIL; status = init_interp_settings(interp, &config); if (_PyStatus_EXCEPTION(status)) { return status; @@ -647,7 +658,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, // XXX For now we do this before the GIL is created. (void) _PyThreadState_SwapNoGIL(tstate); - status = init_interp_create_gil(tstate, config.own_gil); + status = init_interp_create_gil(tstate, config.gil); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -2057,7 +2068,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) goto error; } - status = init_interp_create_gil(tstate, config->own_gil); + status = init_interp_create_gil(tstate, config->gil); if (_PyStatus_EXCEPTION(status)) { goto error; } @@ -2141,6 +2152,9 @@ Py_EndInterpreter(PyThreadState *tstate) // Wrap up existing "threading"-module-created, non-daemon threads. wait_for_thread_shutdown(tstate); + // Make any remaining pending calls. + _Py_FinishPendingCalls(tstate); + _PyAtExit_Call(tstate->interp); if (tstate != interp->threads.head || tstate->next != NULL) { diff --git a/Python/pystate.c b/Python/pystate.c index d63a873a5c3025..da0e1c42d7275f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -380,7 +380,7 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime); _Py_COMP_DIAG_POP -#define NUMLOCKS 8 +#define NUMLOCKS 9 #define LOCKS_INIT(runtime) \ { \ &(runtime)->interpreters.mutex, \ @@ -388,6 +388,7 @@ _Py_COMP_DIAG_POP &(runtime)->getargs.mutex, \ &(runtime)->unicode_state.ids.lock, \ &(runtime)->imports.extensions.mutex, \ + &(runtime)->ceval.pending_mainthread.lock, \ &(runtime)->atexit.mutex, \ &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 05e7b4370869af..99e2eec453ee4b 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1100,15 +1100,13 @@ print_exception_notes(struct exception_print_context *ctx, PyObject *value) return 0; } - if (!PyObject_HasAttr(value, &_Py_ID(__notes__))) { - return 0; - } - PyObject *notes = PyObject_GetAttr(value, &_Py_ID(__notes__)); - if (notes == NULL) { - return -1; + PyObject *notes; + int res = _PyObject_LookupAttr(value, &_Py_ID(__notes__), ¬es); + if (res <= 0) { + return res; } if (!PySequence_Check(notes) || PyUnicode_Check(notes) || PyBytes_Check(notes)) { - int res = 0; + res = 0; if (write_indented_margin(ctx, f) < 0) { res = -1; } diff --git a/Python/specialize.c b/Python/specialize.c index f1684913b1bc30..63b44461007c6e 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1666,9 +1666,9 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, } int argcount = code->co_argcount; int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= argcount); int min_args = argcount-defcount; - if (nargs > argcount || nargs < min_args) { + // GH-105840: min_args is negative when somebody sets too many __defaults__! + if (min_args < 0 || nargs > argcount || nargs < min_args) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } diff --git a/Python/suggestions.c b/Python/suggestions.c index f2c018ef2c4533..ad58393490efc2 100644 --- a/Python/suggestions.c +++ b/Python/suggestions.c @@ -151,15 +151,15 @@ calculate_suggestions(PyObject *dir, } for (int i = 0; i < dir_size; ++i) { PyObject *item = PyList_GET_ITEM(dir, i); + if (_PyUnicode_Equal(name, item)) { + continue; + } Py_ssize_t item_size; const char *item_str = PyUnicode_AsUTF8AndSize(item, &item_size); if (item_str == NULL) { PyMem_Free(buffer); return NULL; } - if (PyUnicode_CompareWithASCIIString(name, item_str) == 0) { - continue; - } // No more than 1/3 of the involved characters should need changed. Py_ssize_t max_distance = (name_size + item_size + 3) * MOVE_COST / 6; // Don't take matches we've already beaten. @@ -220,37 +220,48 @@ get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) assert(code != NULL && code->co_localsplusnames != NULL); PyObject *varnames = _PyCode_GetVarnames(code); + Py_DECREF(code); if (varnames == NULL) { return NULL; } PyObject *dir = PySequence_List(varnames); Py_DECREF(varnames); - Py_DECREF(code); if (dir == NULL) { return NULL; } // Are we inside a method and the instance has an attribute called 'name'? - if (PySequence_Contains(dir, &_Py_ID(self)) > 0) { + int res = PySequence_Contains(dir, &_Py_ID(self)); + if (res < 0) { + goto error; + } + if (res > 0) { PyObject* locals = PyFrame_GetLocals(frame); if (!locals) { goto error; } - PyObject* self = PyDict_GetItem(locals, &_Py_ID(self)); /* borrowed */ - Py_DECREF(locals); + PyObject* self = PyDict_GetItemWithError(locals, &_Py_ID(self)); /* borrowed */ if (!self) { + Py_DECREF(locals); goto error; } - if (PyObject_HasAttr(self, name)) { + PyObject *value; + res = _PyObject_LookupAttr(self, name, &value); + Py_DECREF(locals); + if (res < 0) { + goto error; + } + if (value) { + Py_DECREF(value); Py_DECREF(dir); - return PyUnicode_FromFormat("self.%S", name); + return PyUnicode_FromFormat("self.%U", name); } } PyObject *suggestions = calculate_suggestions(dir, name); Py_DECREF(dir); - if (suggestions != NULL) { + if (suggestions != NULL || PyErr_Occurred()) { return suggestions; } @@ -260,7 +271,7 @@ get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) } suggestions = calculate_suggestions(dir, name); Py_DECREF(dir); - if (suggestions != NULL) { + if (suggestions != NULL || PyErr_Occurred()) { return suggestions; } @@ -319,15 +330,16 @@ offer_suggestions_for_name_error(PyNameErrorObject *exc) assert(frame != NULL); PyObject* suggestion = get_suggestions_for_name_error(name, frame); - bool is_stdlib_module = is_name_stdlib_module(name); - - if (suggestion == NULL && !is_stdlib_module) { + if (suggestion == NULL && PyErr_Occurred()) { return NULL; } // Add a trailer ". Did you mean: (...)?" PyObject* result = NULL; - if (!is_stdlib_module) { + if (!is_name_stdlib_module(name)) { + if (suggestion == NULL) { + return NULL; + } result = PyUnicode_FromFormat(". Did you mean: %R?", suggestion); } else if (suggestion == NULL) { result = PyUnicode_FromFormat(". Did you forget to import %R?", name); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index e6731e7565fcec..4bd38b4b267873 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2277,7 +2277,10 @@ PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void) { char filename[100]; pid_t pid = getpid(); // Use nofollow flag to prevent symlink attacks. - int flags = O_WRONLY | O_CREAT | O_APPEND | O_NOFOLLOW | O_CLOEXEC; + int flags = O_WRONLY | O_CREAT | O_APPEND | O_NOFOLLOW; +#ifdef O_CLOEXEC + flags |= O_CLOEXEC; +#endif snprintf(filename, sizeof(filename) - 1, "/tmp/perf-%jd.map", (intmax_t)pid); int fd = open(filename, flags, 0600); @@ -2718,14 +2721,20 @@ _PySys_AddXOptionWithError(const wchar_t *s) const wchar_t *name_end = wcschr(s, L'='); if (!name_end) { name = PyUnicode_FromWideChar(s, -1); + if (name == NULL) { + goto error; + } value = Py_NewRef(Py_True); } else { name = PyUnicode_FromWideChar(s, name_end - s); + if (name == NULL) { + goto error; + } value = PyUnicode_FromWideChar(name_end + 1, -1); - } - if (name == NULL || value == NULL) { - goto error; + if (value == NULL) { + goto error; + } } if (PyDict_SetItem(opts, name, value) < 0) { goto error; @@ -3365,19 +3374,25 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) static int sys_add_xoption(PyObject *opts, const wchar_t *s) { - PyObject *name, *value; + PyObject *name, *value = NULL; const wchar_t *name_end = wcschr(s, L'='); if (!name_end) { name = PyUnicode_FromWideChar(s, -1); + if (name == NULL) { + goto error; + } value = Py_NewRef(Py_True); } else { name = PyUnicode_FromWideChar(s, name_end - s); + if (name == NULL) { + goto error; + } value = PyUnicode_FromWideChar(name_end + 1, -1); - } - if (name == NULL || value == NULL) { - goto error; + if (value == NULL) { + goto error; + } } if (PyDict_SetItem(opts, name, value) < 0) { goto error; diff --git a/README.rst b/README.rst index bd1c1b06667aa8..5257beacd31854 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.12.0 beta 2 +This is Python version 3.12.0 beta 4 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg diff --git a/Tools/c-analyzer/c_parser/parser/_regexes.py b/Tools/c-analyzer/c_parser/parser/_regexes.py index b7f22b186f4965..5695daff67d6bd 100644 --- a/Tools/c-analyzer/c_parser/parser/_regexes.py +++ b/Tools/c-analyzer/c_parser/parser/_regexes.py @@ -58,6 +58,7 @@ def _ind(text, level=1, edges='both'): extern | register | static | + _Thread_local | typedef | const | @@ -137,7 +138,7 @@ def _ind(text, level=1, edges='both'): ####################################### # variable declarations -_STORAGE = 'auto register static extern'.split() +_STORAGE = 'auto register static extern _Thread_local'.split() STORAGE_CLASS = rf'(?: \b (?: {" | ".join(_STORAGE)} ) \b )' TYPE_QUALIFIER = r'(?: \b (?: const | volatile ) \b )' PTR_QUALIFIER = rf'(?: [*] (?: \s* {TYPE_QUALIFIER} )? )' diff --git a/Tools/c-analyzer/c_parser/preprocessor/common.py b/Tools/c-analyzer/c_parser/preprocessor/common.py index dbe1edeef38527..06f8f4da62b224 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/common.py +++ b/Tools/c-analyzer/c_parser/preprocessor/common.py @@ -1,6 +1,7 @@ import contextlib import distutils.ccompiler import logging +import os import shlex import subprocess import sys @@ -40,7 +41,12 @@ def run_cmd(argv, *, kw.pop('kwargs') kwargs.update(kw) - proc = subprocess.run(argv, **kwargs) + # Remove LANG environment variable: the C parser doesn't support GCC + # localized messages + env = dict(os.environ) + env.pop('LANG', None) + + proc = subprocess.run(argv, env=env, **kwargs) return proc.stdout diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index c680f351f22416..147615707a3cb8 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -219,6 +219,7 @@ def _strip_directives(line, partial=0): line = line[m.end():] line = re.sub(r'__extension__', '', line) + line = re.sub(r'__thread\b', '_Thread_local', line) while (m := COMPILER_DIRECTIVE_RE.match(line)): before, _, _, closed = m.groups() diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 4c40a04a6854bf..afc28e551813b4 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -168,6 +168,12 @@ Modules/_xxinterpchannelsmodule.c - _globals - Python/pyfpe.c - PyFPE_counter - +##----------------------- +## thread-local variables + +Python/import.c - pkgcontext - +Python/pystate.c - _Py_tss_tstate - + ##----------------------- ## should be const # XXX Make them const. @@ -517,6 +523,7 @@ Modules/_testcapimodule.c - g_type_watchers_installed - Modules/_testimportmultiple.c - _barmodule - Modules/_testimportmultiple.c - _foomodule - Modules/_testimportmultiple.c - _testimportmultiple - +Modules/_testinternalcapi.c - pending_identify_result - Modules/_testmultiphase.c - Example_Type_slots - Modules/_testmultiphase.c - Example_Type_spec - Modules/_testmultiphase.c - Example_methods - diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index d182e5e7764e46..04951e97bbd808 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -509,7 +509,7 @@ class PythonLanguage(Language): def permute_left_option_groups(l): """ - Given [1, 2, 3], should yield: + Given [(1,), (2,), (3,)], should yield: () (3,) (2, 3) @@ -524,7 +524,7 @@ def permute_left_option_groups(l): def permute_right_option_groups(l): """ - Given [1, 2, 3], should yield: + Given [(1,), (2,), (3,)], should yield: () (1,) (1, 2) @@ -1966,22 +1966,20 @@ def dump(self): extensions['py'] = PythonLanguage -def file_changed(filename: str, new_contents: str) -> bool: - """Return true if file contents changed (meaning we must update it)""" +def write_file(filename: str, new_contents: str) -> None: try: - with open(filename, encoding="utf-8") as fp: + with open(filename, 'r', encoding="utf-8") as fp: old_contents = fp.read() - return old_contents != new_contents - except FileNotFoundError: - return True - -def write_file(filename: str, new_contents: str): + if old_contents == new_contents: + # no change: avoid modifying the file modification time + return + except FileNotFoundError: + pass # Atomic write using a temporary file and os.replace() filename_new = f"{filename}.new" with open(filename_new, "w", encoding="utf-8") as fp: fp.write(new_contents) - try: os.replace(filename_new, filename) except: @@ -2159,8 +2157,6 @@ def parse(self, input): traceback.format_exc().rstrip()) printer.print_block(block) - clinic_out = [] - # these are destinations not buffers for name, destination in self.destinations.items(): if destination.type == 'suppress': @@ -2168,7 +2164,6 @@ def parse(self, input): output = destination.dump() if output: - block = Block("", dsl_name="clinic", output=output) if destination.type == 'buffer': @@ -2200,11 +2195,10 @@ def parse(self, input): block.input = 'preserve\n' printer_2 = BlockPrinter(self.language) printer_2.print_block(block, core_includes=True) - pair = destination.filename, printer_2.f.getvalue() - clinic_out.append(pair) + write_file(destination.filename, printer_2.f.getvalue()) continue - return printer.f.getvalue(), clinic_out + return printer.f.getvalue() def _module_and_class(self, fields): @@ -2266,14 +2260,9 @@ def parse_file( assert isinstance(language, CLanguage) clinic = Clinic(language, verify=verify, filename=filename) - src_out, clinic_out = clinic.parse(raw) + cooked = clinic.parse(raw) - changes = [(fn, data) for fn, data in clinic_out if file_changed(fn, data)] - if changes: - # Always (re)write the source file. - write_file(output, src_out) - for fn, data in clinic_out: - write_file(fn, data) + write_file(output, cooked) def compute_checksum( @@ -4289,6 +4278,7 @@ def dedent(self, line): StateKeeper = Callable[[str | None], None] +ConverterArgs = dict[str, Any] class DSLParser: def __init__(self, clinic: Clinic) -> None: @@ -5016,10 +5006,10 @@ def bad_node(self, node): key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name self.function.parameters[key] = p - KwargDict = dict[str | None, Any] - @staticmethod - def parse_converter(annotation: ast.expr | None) -> tuple[str, bool, KwargDict]: + def parse_converter( + annotation: ast.expr | None + ) -> tuple[str, bool, ConverterArgs]: match annotation: case ast.Constant(value=str() as value): return value, True, {} @@ -5027,10 +5017,11 @@ def parse_converter(annotation: ast.expr | None) -> tuple[str, bool, KwargDict]: return name, False, {} case ast.Call(func=ast.Name(name)): symbols = globals() - kwargs = { - node.arg: eval_ast_expr(node.value, symbols) - for node in annotation.keywords - } + kwargs: ConverterArgs = {} + for node in annotation.keywords: + if not isinstance(node.arg, str): + fail("Cannot use a kwarg splat in a function-call annotation") + kwargs[node.arg] = eval_ast_expr(node.value, symbols) return name, False, kwargs case _: fail( diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index f6a5adb4519fdb..92e97cb261719c 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -153,7 +153,7 @@ def prepare(script=None, outdir=None): print(f'configuring python in {builddir}...') cmd = [ os.path.join(srcdir, 'configure'), - *shlex.split(get_config_var(srcdir, 'CONFIG_ARGS') or ''), + *shlex.split(get_config_var(SRCDIR, 'CONFIG_ARGS') or ''), ] ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache')) prefix = os.path.join(outdir, 'python-installation') diff --git a/Tools/importbench/importbench.py b/Tools/importbench/importbench.py index 619263b553c081..0c4b3bc73517c5 100644 --- a/Tools/importbench/importbench.py +++ b/Tools/importbench/importbench.py @@ -15,6 +15,7 @@ import sys import tabnanny import timeit +import types def bench(name, cleanup=lambda: None, *, seconds=1, repeat=3): @@ -40,7 +41,7 @@ def bench(name, cleanup=lambda: None, *, seconds=1, repeat=3): def from_cache(seconds, repeat): """sys.modules""" name = '' - module = imp.new_module(name) + module = types.ModuleType(name) module.__file__ = '' module.__package__ = '' with util.uncache(name): diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 5805ff63717440..aace684045b9f8 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -1,4 +1,5 @@ import itertools +import os import pathlib import sys import sysconfig @@ -27,6 +28,46 @@ def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[ return f"{flags} {py_flags_nodist}".split() +def fixup_build_ext(cmd): + """Function needed to make build_ext tests pass. + + When Python was built with --enable-shared on Unix, -L. is not enough to + find libpython.so, because regrtest runs in a tempdir, not in the + source directory where the .so lives. + + When Python was built with in debug mode on Windows, build_ext commands + need their debug attribute set, and it is not done automatically for + some reason. + + This function handles both of these things. Example use: + + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.ensure_finalized() + + Unlike most other Unix platforms, Mac OS X embeds absolute paths + to shared libraries into executables, so the fixup is not needed there. + + Taken from distutils (was part of the CPython stdlib until Python 3.11) + """ + if os.name == 'nt': + cmd.debug = sys.executable.endswith('_d.exe') + elif sysconfig.get_config_var('Py_ENABLE_SHARED'): + # To further add to the shared builds fun on Unix, we can't just add + # library_dirs to the Extension() instance because that doesn't get + # plumbed through to the final compiler command. + runshared = sysconfig.get_config_var('RUNSHARED') + if runshared is None: + cmd.library_dirs = ['.'] + else: + if sys.platform == 'darwin': + cmd.library_dirs = [] + else: + name, equals, value = runshared.partition('=') + cmd.library_dirs = [d for d in value.split(os.pathsep) if d] + + + def compile_c_extension( generated_source_path: str, build_dir: Optional[str] = None, @@ -49,16 +90,15 @@ def compile_c_extension( static library of the common parser sources (this is useful in case you are creating multiple extensions). """ - import distutils.log - from distutils.core import Distribution, Extension - from distutils.tests.support import fixup_build_ext # type: ignore + import setuptools.logging - from distutils.ccompiler import new_compiler - from distutils.dep_util import newer_group - from distutils.sysconfig import customize_compiler + from setuptools import Extension, Distribution + from setuptools._distutils.dep_util import newer_group + from setuptools._distutils.ccompiler import new_compiler + from setuptools._distutils.sysconfig import customize_compiler if verbose: - distutils.log.set_threshold(distutils.log.DEBUG) + setuptools.logging.set_threshold(setuptools.logging.logging.DEBUG) source_file_path = pathlib.Path(generated_source_path) extension_name = source_file_path.stem diff --git a/Tools/wasm/.editorconfig b/Tools/wasm/.editorconfig index da1aa6acaccc7e..4de5fe5954d84b 100644 --- a/Tools/wasm/.editorconfig +++ b/Tools/wasm/.editorconfig @@ -1,4 +1,5 @@ -root = false # This extends the root .editorconfig +# This extends the root .editorconfig +root = false [*.{html,js}] trim_trailing_whitespace = true diff --git a/configure b/configure index e89155b5bbef94..248b14e254f6db 100755 --- a/configure +++ b/configure @@ -27291,7 +27291,6 @@ SRCDIRS="\ Modules/_hacl \ Modules/_io \ Modules/_multiprocessing \ - Modules/_sha3 \ Modules/_sqlite \ Modules/_sre \ Modules/_testcapi \ diff --git a/configure.ac b/configure.ac index 14354a07ab5073..96c96e4b0d735b 100644 --- a/configure.ac +++ b/configure.ac @@ -6697,7 +6697,6 @@ SRCDIRS="\ Modules/_hacl \ Modules/_io \ Modules/_multiprocessing \ - Modules/_sha3 \ Modules/_sqlite \ Modules/_sre \ Modules/_testcapi \