Merge branch 'main' into unpickle-overallocate

serhiy-storchaka · Jun 29, 2024 · 3462d0e · 3462d0e
2 parents d0e667e + 2cb84b1
commit 3462d0e
Show file tree

Hide file tree

Showing 1,075 changed files with 41,610 additions and 21,563 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -13,6 +13,8 @@
 
 # Build system
 configure*                    @erlend-aasland @corona10
+Makefile.pre.in               @erlend-aasland
+Modules/Setup*                @erlend-aasland
 
 # asyncio
 **/*asyncio*                  @1st1 @asvetlov @gvanrossum @kumaraditya303 @willingc
@@ -29,19 +31,23 @@ Objects/type*                 @markshannon
 Objects/codeobject.c          @markshannon
 Objects/frameobject.c         @markshannon
 Objects/call.c                @markshannon
-Python/ceval*.c               @markshannon @gvanrossum
-Python/ceval*.h               @markshannon @gvanrossum
+Python/ceval*.c               @markshannon
+Python/ceval*.h               @markshannon
 Python/compile.c              @markshannon @iritkatriel
 Python/assemble.c             @markshannon @iritkatriel
 Python/flowgraph.c            @markshannon @iritkatriel
+Python/instruction_sequence.c @iritkatriel
 Python/ast_opt.c              @isidentical
-Python/bytecodes.c            @markshannon @gvanrossum
-Python/optimizer*.c           @markshannon @gvanrossum
+Python/bytecodes.c            @markshannon
+Python/optimizer*.c           @markshannon
 Python/optimizer_analysis.c   @Fidget-Spinner
 Python/optimizer_bytecodes.c  @Fidget-Spinner
+Python/symtable.c             @JelleZijlstra @carljm
+Lib/_pyrepl/*                 @pablogsal @lysnikolaou @ambv
 Lib/test/test_patma.py        @brandtbucher
 Lib/test/test_type_*.py       @JelleZijlstra
-Lib/test/test_capi/test_misc.py  @markshannon @gvanrossum
+Lib/test/test_capi/test_misc.py  @markshannon
+Lib/test/test_pyrepl/*        @pablogsal @lysnikolaou @ambv
 Tools/c-analyzer/             @ericsnowcurrently
 
 # dbm
@@ -72,11 +78,8 @@ Programs/python.c             @ericsnowcurrently
 Tools/build/generate_global_objects.py  @ericsnowcurrently
 
 # Exceptions
-Lib/traceback.py              @iritkatriel
 Lib/test/test_except*.py      @iritkatriel
-Lib/test/test_traceback.py    @iritkatriel
 Objects/exceptions.c          @iritkatriel
-Python/traceback.c            @iritkatriel
 
 # Hashing
 **/*hashlib*                  @gpshead @tiran
@@ -150,13 +153,13 @@ Include/internal/pycore_time.h  @pganssle @abalkin
 /Lib/test/test_tokenize.py    @pablogsal @lysnikolaou
 
 # Code generator
-/Tools/cases_generator/        @gvanrossum
+/Tools/cases_generator/        @markshannon
 
 # AST
-Python/ast.c                  @isidentical
-Parser/asdl.py                @isidentical
-Parser/asdl_c.py              @isidentical
-Lib/ast.py                    @isidentical
+Python/ast.c                  @isidentical @JelleZijlstra
+Parser/asdl.py                @isidentical @JelleZijlstra
+Parser/asdl_c.py              @isidentical @JelleZijlstra
+Lib/ast.py                    @isidentical @JelleZijlstra
 
 # Mock
 /Lib/unittest/mock.py         @cjw296
@@ -173,6 +176,10 @@ Lib/ast.py                    @isidentical
 /Lib/test/test_subprocess.py  @gpshead
 /Modules/*subprocess*         @gpshead
 
+# debugger
+**/*pdb*                      @gaogaotiantian
+**/*bdb*                      @gaogaotiantian
+
 # Limited C API & stable ABI
 Tools/build/stable_abi.py     @encukou
 Misc/stable_abi.toml          @encukou
@@ -194,7 +201,6 @@ Doc/c-api/stable.rst          @encukou
 **/*itertools*                @rhettinger
 **/*collections*              @rhettinger
 **/*random*                   @rhettinger
-**/*queue*                    @rhettinger
 **/*bisect*                   @rhettinger
 **/*heapq*                    @rhettinger
 **/*functools*                @rhettinger
@@ -205,6 +211,7 @@ Doc/c-api/stable.rst          @encukou
 **/*ensurepip*                @pfmoore @pradyunsg
 
 **/*idlelib*                  @terryjreedy
+/Doc/library/idle.rst         @terryjreedy
 
 **/*typing*                   @JelleZijlstra @AlexWaygood
 
@@ -240,7 +247,7 @@ Doc/howto/clinic.rst          @erlend-aasland
 **/*interpreteridobject.*     @ericsnowcurrently
 **/*crossinterp*              @ericsnowcurrently
 Lib/test/support/interpreters/  @ericsnowcurrently
-Modules/_xx*interp*module.c   @ericsnowcurrently
+Modules/_interp*module.c      @ericsnowcurrently
 Lib/test/test_interpreters/   @ericsnowcurrently
 
 # Android

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -54,7 +54,7 @@ jobs:
             # into the PR branch anyway.
             #
             # https://github.com/python/core-workflow/issues/373
-            git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
+            git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
           fi
 
           # Check if we should run hypothesis tests
@@ -199,8 +199,9 @@ jobs:
     uses: ./.github/workflows/reusable-macos.yml
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
-      # macos-14 is M1, macos-13 is Intel
-      os-matrix: '["macos-14", "macos-13"]'
+      # Cirrus and macos-14 are M1, macos-13 is default GHA Intel.
+      # Cirrus used for upstream, macos-14 for forks.
+      os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-14", "macos-13"]'
 
   build_macos_free_threading:
     name: 'macOS (free-threading)'
@@ -210,8 +211,9 @@ jobs:
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
       free-threading: true
-      # macos-14-large is Intel with 12 cores (most parallelism)
-      os-matrix: '["macos-14"]'
+      # Cirrus and macos-14 are M1.
+      # Cirrus used for upstream, macos-14 for forks.
+      os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-14"]'
 
   build_ubuntu:
     name: 'Ubuntu'
@@ -388,7 +390,7 @@ jobs:
       id: cache-hypothesis-database
       uses: actions/cache@v4
       with:
-        path: ./hypothesis
+        path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/
         key: hypothesis-database-${{ github.head_ref || github.run_id }}
         restore-keys: |
           - hypothesis-database-
@@ -416,7 +418,7 @@ jobs:
       if: always()
       with:
         name: hypothesis-example-db
-        path: .hypothesis/examples/
+        path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/
 
 
   build_asan:

diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml
@@ -26,8 +26,22 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  interpreter:
+    name: Interpreter (Debug)
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build tier two interpreter
+        run: |
+          ./configure --enable-experimental-jit=interpreter --with-pydebug
+          make all --jobs 4
+      - name: Test tier two interpreter
+        run: |
+          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
   jit:
     name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }})
+    needs: interpreter
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 90
     strategy:
@@ -119,17 +133,15 @@ jobs:
           make all --jobs 4
           ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-      # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Native Linux
         if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-      # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Emulated Linux
         if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
         # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux.
@@ -147,6 +159,25 @@ jobs:
           CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
             CPP="$CC --preprocess" \
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
-            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
           ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+
+  jit-with-disabled-gil:
+    name: Free-Threaded (Debug)
+    needs: interpreter
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Build with JIT enabled and GIL disabled
+        run: |
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 18
+          export PATH="$(llvm-config-18 --bindir):$PATH"
+          ./configure --enable-experimental-jit --with-pydebug --disable-gil
+          make all --jobs 4
+      - name: Run tests
+        run: |
+          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
@@ -34,6 +34,7 @@ concurrency:
 jobs:
   mypy:
     strategy:
+      fail-fast: false
       matrix:
         target: [
           "Lib/_pyrepl",

diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml
@@ -62,7 +62,8 @@ jobs:
         python Doc/tools/check-warnings.py \
           --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \
           --fail-if-regression \
-          --fail-if-improved
+          --fail-if-improved \
+          --fail-if-new-news-nit
 
   # This build doesn't use problem matchers or check annotations
   build_doc_oldest_supported_sphinx:

diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml
@@ -14,7 +14,7 @@ on:
 
 jobs:
   build_macos:
-    name: 'build and test'
+    name: build and test (${{ matrix.os }})
     timeout-minutes: 60
     env:
       HOMEBREW_NO_ANALYTICS: 1
@@ -27,6 +27,13 @@ jobs:
       fail-fast: false
       matrix:
         os: ${{fromJson(inputs.os-matrix)}}
+        is-fork:
+          - ${{ github.repository_owner != 'python' }}
+        exclude:
+          - os: "ghcr.io/cirruslabs/macos-runner:sonoma"
+            is-fork: true
+          - os: "macos-14"
+            is-fork: false
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -26,6 +26,9 @@ build:
         exit 183;
       fi
 
+    - asdf plugin add uv
+    - asdf install uv latest
+    - asdf global uv latest
     - make -C Doc venv html
     - mkdir _readthedocs
     - mv Doc/build/html _readthedocs/html

diff --git a/Doc/Makefile b/Doc/Makefile
@@ -150,10 +150,14 @@ gettext: build
 htmlview: html
 	$(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))"
 
+.PHONY: ensure-sphinx-autobuild
+ensure-sphinx-autobuild: venv
+	$(call ensure_package,sphinx-autobuild)
+
 .PHONY: htmllive
 htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild
 htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0
-htmllive: html
+htmllive: ensure-sphinx-autobuild html
 
 .PHONY: clean
 clean: clean-venv
@@ -170,10 +174,15 @@ venv:
 		echo "To recreate it, remove it first with \`make clean-venv'."; \
 	else \
 		echo "Creating venv in $(VENVDIR)"; \
-		$(PYTHON) -m venv $(VENVDIR); \
-		$(VENVDIR)/bin/python3 -m pip install --upgrade pip; \
-		$(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \
-		echo "The venv has been created in the $(VENVDIR) directory"; \
+		if uv --version > /dev/null; then \
+			uv venv $(VENVDIR); \
+			VIRTUAL_ENV=$(VENVDIR) uv pip install -r $(REQUIREMENTS); \
+		else \
+			$(PYTHON) -m venv $(VENVDIR); \
+			$(VENVDIR)/bin/python3 -m pip install --upgrade pip; \
+			$(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \
+			echo "The venv has been created in the $(VENVDIR) directory"; \
+		fi; \
 	fi
 
 .PHONY: dist
@@ -231,9 +240,17 @@ dist:
 	rm -r dist/python-$(DISTVERSION)-docs-texinfo
 	rm dist/python-$(DISTVERSION)-docs-texinfo.tar
 
+define ensure_package
+	if uv --version > /dev/null; then \
+		$(VENVDIR)/bin/python3 -m $(1) --version > /dev/null || VIRTUAL_ENV=$(VENVDIR) uv pip install $(1); \
+	else \
+		$(VENVDIR)/bin/python3 -m $(1) --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install $(1); \
+	fi
+endef
+
 .PHONY: check
 check: venv
-	$(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit
+	$(call ensure_package,pre_commit)
 	$(VENVDIR)/bin/python3 -m pre_commit run --all-files
 
 .PHONY: serve

diff --git a/Doc/README.rst b/Doc/README.rst
@@ -28,7 +28,7 @@ install the tools into there.
 Using make
 ----------
 
-To get started on UNIX, you can create a virtual environment and build
+To get started on Unix, you can create a virtual environment and build
 documentation with the commands::
 
   make venv
@@ -40,13 +40,13 @@ If you'd like to create the virtual environment in a different location,
 you can specify it using the ``VENVDIR`` variable.
 
 You can also skip creating the virtual environment altogether, in which case
-the Makefile will look for instances of ``sphinx-build`` and ``blurb``
+the ``Makefile`` will look for instances of ``sphinx-build`` and ``blurb``
 installed on your process ``PATH`` (configurable with the ``SPHINXBUILD`` and
 ``BLURB`` variables).
 
-On Windows, we try to emulate the Makefile as closely as possible with a
+On Windows, we try to emulate the ``Makefile`` as closely as possible with a
 ``make.bat`` file. If you need to specify the Python interpreter to use,
-set the PYTHON environment variable.
+set the ``PYTHON`` environment variable.
 
 Available make targets are:
 
@@ -62,15 +62,19 @@ Available make targets are:
 * "htmlview", which re-uses the "html" builder, but then opens the main page
   in your default web browser.
 
+* "htmllive", which re-uses the "html" builder, rebuilds the docs,
+  starts a local server, and automatically reloads the page in your browser
+  when you make changes to reST files (Unix only).
+
 * "htmlhelp", which builds HTML files and a HTML Help project file usable to
   convert them into a single Compiled HTML (.chm) file -- these are popular
   under Microsoft Windows, but very handy on every platform.
 
   To create the CHM file, you need to run the Microsoft HTML Help Workshop
-  over the generated project (.hhp) file.  The make.bat script does this for
+  over the generated project (.hhp) file.  The ``make.bat`` script does this for
   you on Windows.
 
-* "latex", which builds LaTeX source files as input to "pdflatex" to produce
+* "latex", which builds LaTeX source files as input to ``pdflatex`` to produce
   PDF documents.
 
 * "text", which builds a plain text file for each source file.
@@ -95,8 +99,6 @@ Available make targets are:
 
 * "check", which checks for frequent markup errors.
 
-* "serve", which serves the build/html directory on port 8000.
-
 * "dist", (Unix only) which creates distributable archives of HTML, text,
   PDF, and EPUB builds.
 

diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
@@ -147,9 +147,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`.
       or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard
       :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``.
 
-   .. c:member:: const char *format
+   .. c:member:: char *format
 
-      A *NUL* terminated string in :mod:`struct` module style syntax describing
+      A *NULL* terminated string in :mod:`struct` module style syntax describing
       the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes)
       is assumed.