Merge branch '3.9' into backport-77e0919-3.9

.github/workflows/build.yml

@@ -25,7 +25,7 @@ jobs:
       run_tests: ${{ steps.check.outputs.run_tests }}
       run_ssl_tests: ${{ steps.check.outputs.run_ssl_tests }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Check for source changes
         id: check
         run: |
@@ -57,8 +57,8 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
       - name: Install Dependencies
         run: |
             sudo ./.github/workflows/posix-deps-apt.sh
@@ -79,8 +79,8 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
       - name: Add ccache to PATH
@@ -93,8 +93,12 @@ jobs:
           grep "aclocal 1.16.3" aclocal.m4
           grep -q "runstatedir" configure
           grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
+      - name: Configure CPython
+        run: |
+          # Build Python with the libpython dynamic library
+          ./configure --config-cache --with-pydebug --enable-shared
       - name: Regenerate autoconf files
-        run: docker run --rm -v $(pwd):/src quay.io/tiran/cpython_autoconf:269
+        run: make regen-configure
       - name: Build CPython
         run: |
           ./configure --with-pydebug
@@ -119,7 +123,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build CPython
       run: .\PCbuild\build.bat -e -p Win32
     - name: Display build info
@@ -133,7 +137,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build CPython
       run: .\PCbuild\build.bat -e -p x64
     - name: Display build info
@@ -152,7 +156,7 @@ jobs:
       HOMEBREW_NO_INSTALL_CLEANUP: 1
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install Homebrew dependencies
       run: brew install pkg-config [email protected] xz gdbm tcl-tk
     - name: Configure CPython
@@ -178,10 +182,10 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1v
+      OPENSSL_VER: 3.0.11
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Configure OpenSSL env vars
@@ -220,14 +224,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        openssl_ver: [1.0.2u, 1.1.0l, 1.1.1v, 3.0.10, 3.1.2]
+        openssl_ver: [1.0.2u, 1.1.0l, 1.1.1w, 3.0.11, 3.1.3]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
       OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}
       LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Configure OpenSSL env vars

.github/workflows/build_msi.yml

@@ -25,14 +25,14 @@ jobs:
     name: 'Windows (x86) Installer'
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build CPython installer
       run: .\Tools\msi\build.bat -x86
 
   build_win_amd64:
     name: 'Windows (x64) Installer'
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build CPython installer
       run: .\Tools\msi\build.bat -x64

.github/workflows/coverage.yml

@@ -27,7 +27,7 @@ jobs:
     env:
       OPENSSL_VER: 1.1.1k
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: 'Restore OpenSSL build'
@@ -76,7 +76,7 @@ jobs:
     name: 'Ubuntu (C Coverage)'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Configure CPython

.github/workflows/doc.yml

@@ -24,7 +24,7 @@ jobs:
     name: 'Docs'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: 'Install Dependencies'
       run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican
     - name: 'Configure CPython'

.github/workflows/verify-ensurepip-wheels.yml

@@ -20,7 +20,7 @@ jobs:
   verify:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: '3'

.readthedocs.yml

@@ -0,0 +1,12 @@
+# This is a dummy config file so that readthedocs.org doesn't fail on security branches.
+# Note that this won't result in docs actually getting built;
+# clicking on the docs preview link on a PR will result in a 404.
+version: 2
+formats: []
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.11"
+  jobs:
+    post_checkout:
+    - exit 183

Doc/constraints.txt

@@ -0,0 +1,24 @@
+# We have upper bounds on our transitive dependencies here
+# To avoid new releases unexpectedly breaking our build.
+# This file can be updated on an ad-hoc basis,
+# though it will probably have to be updated
+# whenever Doc/requirements.txt is updated.
+
+# Direct dependencies of Sphinx
+babel<3
+colorama<0.5
+imagesize<1.5
+Jinja2<3.2
+packaging<24
+Pygments>=2.16.1,<3
+requests<3
+snowballstemmer<3
+sphinxcontrib-applehelp<1.0.5
+sphinxcontrib-devhelp<1.0.6
+sphinxcontrib-htmlhelp<2.0.5
+sphinxcontrib-jsmath<1.1
+sphinxcontrib-qthelp<1.0.7
+sphinxcontrib-serializinghtml<1.1.10
+
+# Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above)
+MarkupSafe<2.2

Doc/requirements.txt

@@ -10,9 +10,13 @@ sphinx==2.4.4
 docutils==0.17.1
 # Jinja version is pinned to a version compatible with Sphinx version 2.4.4.
 jinja2==3.0.3
+# Alabaster version is pinned to a version compatible with Sphinx version 2.4.4.
+alabaster==0.7.13
 
 blurb
 
 # The theme used by the documentation is stored separately, so we need
 # to install that as well.
 python-docs-theme>=2022.1
+
+-c constraints.txt

Include/patchlevel.h

@@ -18,12 +18,12 @@
 /*--start constants--*/
 #define PY_MAJOR_VERSION        3
 #define PY_MINOR_VERSION        9
-#define PY_MICRO_VERSION        17
+#define PY_MICRO_VERSION        18
 #define PY_RELEASE_LEVEL        PY_RELEASE_LEVEL_FINAL
 #define PY_RELEASE_SERIAL       0
 
 /* Version as a string */
-#define PY_VERSION              "3.9.17+"
+#define PY_VERSION              "3.9.18+"
 /*--end constants--*/
 
 /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.

Lib/site.py

@@ -74,6 +74,7 @@
 import builtins
 import _sitebuiltins
 import io
+import stat
 
 # Prefixes for site-packages; add additional prefixes like /usr/local here
 PREFIXES = [sys.prefix, sys.exec_prefix]
@@ -156,6 +157,13 @@ def addpackage(sitedir, name, known_paths):
     else:
         reset = False
     fullname = os.path.join(sitedir, name)
+    try:
+        st = os.lstat(fullname)
+    except OSError:
+        return
+    if ((getattr(st, 'st_flags', 0) & stat.UF_HIDDEN) or
+        (getattr(st, 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_HIDDEN)):
+        return
     try:
         f = io.TextIOWrapper(io.open_code(fullname))
     except OSError:
@@ -203,7 +211,8 @@ def addsitedir(sitedir, known_paths=None):
         names = os.listdir(sitedir)
     except OSError:
         return
-    names = [name for name in names if name.endswith(".pth")]
+    names = [name for name in names
+             if name.endswith(".pth") and not name.startswith(".")]
     for name in sorted(names):
         addpackage(sitedir, name, known_paths)
     if reset:

Lib/ssl.py

@@ -1049,7 +1049,11 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True,
                     self.close()
                 except OSError:
                     pass
-                raise notconn_pre_handshake_data_error
+                try:
+                    raise notconn_pre_handshake_data_error
+                finally:
+                    # Explicitly break the reference cycle.
+                    notconn_pre_handshake_data_error = None
         else:
             connected = True
 

Lib/tempfile.py

@@ -268,6 +268,22 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type):
     raise FileExistsError(_errno.EEXIST,
                           "No usable temporary file name found")
 
+def _dont_follow_symlinks(func, path, *args):
+    # Pass follow_symlinks=False, unless not supported on this platform.
+    if func in _os.supports_follow_symlinks:
+        func(path, *args, follow_symlinks=False)
+    elif _os.name == 'nt' or not _os.path.islink(path):
+        func(path, *args)
+
+def _resetperms(path):
+    try:
+        chflags = _os.chflags
+    except AttributeError:
+        pass
+    else:
+        _dont_follow_symlinks(chflags, path, 0)
+    _dont_follow_symlinks(_os.chmod, path, 0o700)
+
 
 # User visible interfaces.
 
@@ -789,17 +805,10 @@ def __init__(self, suffix=None, prefix=None, dir=None):
     def _rmtree(cls, name):
         def onerror(func, path, exc_info):
             if issubclass(exc_info[0], PermissionError):
-                def resetperms(path):
-                    try:
-                        _os.chflags(path, 0)
-                    except AttributeError:
-                        pass
-                    _os.chmod(path, 0o700)
-
                 try:
                     if path != name:
-                        resetperms(_os.path.dirname(path))
-                    resetperms(path)
+                        _resetperms(_os.path.dirname(path))
+                    _resetperms(path)
 
                     try:
                         _os.unlink(path)

Lib/test/test_codecencodings_iso2022.py

@@ -24,6 +24,52 @@ class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase):
         (b'ab\x1BNdef', 'replace', 'abdef'),
     )
 
+class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase):
+    encoding = 'iso2022_jp_3'
+    tstring = multibytecodec_support.load_teststring('iso2022_jp')
+    codectests = COMMON_CODEC_TESTS + (
+        (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'),
+        (b'\x1B$(O\x2E\x23\x1B(B', 'strict', '\u3402'      ),
+        (b'\x1B$(O\x2E\x22\x1B(B', 'strict', '\U0002000B'  ),
+        (b'\x1B$(O\x24\x77\x1B(B', 'strict', '\u304B\u309A'),
+        (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02'      ),
+        (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2'  ),
+        ('\u3402',       'strict', b'\x1B$(O\x2E\x23\x1B(B'),
+        ('\U0002000B',   'strict', b'\x1B$(O\x2E\x22\x1B(B'),
+        ('\u304B\u309A', 'strict', b'\x1B$(O\x24\x77\x1B(B'),
+        ('\u4E02',       'strict', b'\x1B$(P\x21\x22\x1B(B'),
+        ('\U0002A6B2',   'strict', b'\x1B$(P\x7E\x76\x1B(B'),
+        (b'ab\x1B$(O\x2E\x21\x1B(Bdef', 'replace', 'ab\uFFFDdef'),
+        ('ab\u4FF1def', 'replace', b'ab?def'),
+    )
+    xmlcharnametest = (
+        '\xAB\u211C\xBB = \u2329\u1234\u232A',
+        b'\x1B$(O\x29\x28\x1B(Bℜ\x1B$(O\x29\x32\x1B(B = ⟨ሴ⟩'
+    )
+
+class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase):
+    encoding = 'iso2022_jp_2004'
+    tstring = multibytecodec_support.load_teststring('iso2022_jp')
+    codectests = COMMON_CODEC_TESTS + (
+        (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'),
+        (b'\x1B$(Q\x2E\x23\x1B(B', 'strict', '\u3402'      ),
+        (b'\x1B$(Q\x2E\x22\x1B(B', 'strict', '\U0002000B'  ),
+        (b'\x1B$(Q\x24\x77\x1B(B', 'strict', '\u304B\u309A'),
+        (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02'      ),
+        (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2'  ),
+        ('\u3402',       'strict', b'\x1B$(Q\x2E\x23\x1B(B'),
+        ('\U0002000B',   'strict', b'\x1B$(Q\x2E\x22\x1B(B'),
+        ('\u304B\u309A', 'strict', b'\x1B$(Q\x24\x77\x1B(B'),
+        ('\u4E02',       'strict', b'\x1B$(P\x21\x22\x1B(B'),
+        ('\U0002A6B2',   'strict', b'\x1B$(P\x7E\x76\x1B(B'),
+        (b'ab\x1B$(Q\x2E\x21\x1B(Bdef', 'replace', 'ab\u4FF1def'),
+        ('ab\u4FF1def', 'replace', b'ab\x1B$(Q\x2E\x21\x1B(Bdef'),
+    )
+    xmlcharnametest = (
+        '\xAB\u211C\xBB = \u2329\u1234\u232A',
+        b'\x1B$(Q\x29\x28\x1B(Bℜ\x1B$(Q\x29\x32\x1B(B = ⟨ሴ⟩'
+    )
+
 class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'iso2022_kr'
     tstring = multibytecodec_support.load_teststring('iso2022_kr')

Lib/test/test_httpservers.py

@@ -418,10 +418,10 @@ def test_undecodable_filename(self):
     def test_undecodable_parameter(self):
         # sanity check using a valid parameter
         response = self.request(self.base_url + '/?x=123').read()
-        self.assertRegex(response, f'listing for {self.base_url}/\?x=123'.encode('latin1'))
+        self.assertRegex(response, f'listing for {self.base_url}/\\?x=123'.encode('latin1'))
         # now the bogus encoding
         response = self.request(self.base_url + '/?x=%bb').read()
-        self.assertRegex(response, f'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1'))
+        self.assertRegex(response, f'listing for {self.base_url}/\\?x=\xef\xbf\xbd'.encode('latin1'))
 
     def test_get_dir_redirect_location_domain_injection_bug(self):
         """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location.

Lib/test/test_mmap.py

@@ -240,10 +240,15 @@ def test_access_parameter(self):
             # Try writing with PROT_EXEC and without PROT_WRITE
             prot = mmap.PROT_READ | getattr(mmap, 'PROT_EXEC', 0)
             with open(TESTFN, "r+b") as f:
-                m = mmap.mmap(f.fileno(), mapsize, prot=prot)
-                self.assertRaises(TypeError, m.write, b"abcdef")
-                self.assertRaises(TypeError, m.write_byte, 0)
-                m.close()
+                try:
+                    m = mmap.mmap(f.fileno(), mapsize, prot=prot)
+                except PermissionError:
+                    # on macOS 14, PROT_READ | PROT_WRITE is not allowed
+                    pass
+                else:
+                    self.assertRaises(TypeError, m.write, b"abcdef")
+                    self.assertRaises(TypeError, m.write_byte, 0)
+                    m.close()
 
     def test_bad_file_desc(self):
         # Try opening a bad file descriptor...