deps(python): enable Python 3.13t support

.clang-format

@@ -15,13 +15,13 @@ BreakBeforeTernaryOperators: true
 FixNamespaceComments: true
 IncludeBlocks: Regroup
 IncludeCategories:
-  - Regex: '^(<|")Python\.h("|>)$'
+  - Regex: '^[<"]Python\.h[">]$'
     Priority: 2
     CaseSensitive: true
-  - Regex: '^(<|")(pybind11/.*)("|>)$'
+  - Regex: '^[<"]pybind11/.*[">]$'
     Priority: 3
     CaseSensitive: true
-  - Regex: '^<[[:alnum:]_]+(\.h)?>$'
+  - Regex: '^<[[:alnum:]_/]+(\.h)?>$'
     Priority: 1
   - Regex: '^"include/'
     Priority: 4

.clang-tidy

@@ -22,6 +22,6 @@ readability-*,
 -readability-identifier-length,
 '
 CheckOptions:
-    misc-include-cleaner.IgnoreHeaders: 'python.*/.*;pybind11/.*'
+    misc-include-cleaner.IgnoreHeaders: 'python.*/.*;pybind11/.*;include/.*'
 HeaderFilterRegex: '^include/.*$'
 ...

.github/workflows/build.yml

@@ -147,7 +147,7 @@ jobs:
           - os: macos-latest
             python-version: "3.7"
       fail-fast: false
-    timeout-minutes: 120
+    timeout-minutes: 180
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/set_cibw_build.py

@@ -7,7 +7,8 @@
 
 
 MAJOR, MINOR, *_ = platform.python_version_tuple()
-CIBW_BUILD = f'CIBW_BUILD=*{platform.python_implementation().lower()[0]}p{MAJOR}{MINOR}-*'
+IMPLEMENTATION = platform.python_implementation()
+CIBW_BUILD = f'CIBW_BUILD=*{IMPLEMENTATION.lower()[0]}p{MAJOR}{MINOR}{{,?}}-*'
 
 print(CIBW_BUILD)
 with open(os.getenv('GITHUB_ENV'), mode='a', encoding='utf-8') as file:

.github/workflows/tests-with-pydebug.yml

@@ -37,9 +37,25 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
-        python-abiflags: ["d"]
+        python-abiflags: ["d", "td"]
+        exclude:
+          - python-version: "3.7"
+            python-abiflags: "td"
+          - python-version: "3.8"
+            python-abiflags: "td"
+          - python-version: "3.9"
+            python-abiflags: "td"
+          - python-version: "3.10"
+            python-abiflags: "td"
+          - python-version: "3.11"
+            python-abiflags: "td"
+          - python-version: "3.12"
+            python-abiflags: "td"
+          - os: windows-latest  # pyenv-win does not support Python 3.13t yet
+            python-version: "3.13"
+            python-abiflags: "td"
       fail-fast: false
-    timeout-minutes: 60
+    timeout-minutes: 90
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -65,7 +81,12 @@ jobs:
         shell: bash
         run: |
           pyenv install --list
-          if ! PYTHON_VERSION="$(pyenv latest --known "${{ matrix.python-version }}")"; then
+          if [[ "${{ matrix.python-abiflags }}" == *t* ]]; then
+            PYTHON_VERSION="$(
+              pyenv install --list | tr -d ' ' | grep -E "^${{ matrix.python-version }}" |
+                grep -vF '-' | grep -E '[0-9]t$' | sort -rV | head -n 1
+            )"
+          elif ! PYTHON_VERSION="$(pyenv latest --known "${{ matrix.python-version }}")"; then
             PYTHON_VERSION="$(
               pyenv install --list | tr -d ' ' | grep -E "^${{ matrix.python-version }}" |
                 grep -vF '-' | grep -E '[0-9]$' | sort -rV | head -n 1
@@ -109,4 +130,4 @@ jobs:
 
       - name: Test with pytest
         run: |
-          make test PYTESTOPTS="--exitfirst --verbosity=0 --durations=10"
+          make test PYTESTOPTS="--exitfirst"

.github/workflows/tests.yml

@@ -46,7 +46,7 @@ jobs:
           - os: macos-latest
             python-version: "3.7"  # Python 3.7 does not support macOS ARM64
       fail-fast: false
-    timeout-minutes: 60
+    timeout-minutes: 90
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.pre-commit-config.yaml

@@ -30,7 +30,7 @@ repos:
     hooks:
       - id: clang-format
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.7
+    rev: v0.6.8
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add Python 3.13t support by [@XuehaiPan](https://github.com/XuehaiPan) in [#137](https://github.com/metaopt/optree/pull/137).
 - Expose Python implementation for C utilities for `namedtuple` and `PyStructSequence` by [@XuehaiPan](https://github.com/XuehaiPan) in [#157](https://github.com/metaopt/optree/pull/157).
 - Add `dataclasses` integration by [@XuehaiPan](https://github.com/XuehaiPan) in [#142](https://github.com/metaopt/optree/pull/142).
 - Add Python 3.13 support by [@XuehaiPan](https://github.com/XuehaiPan) in [#156](https://github.com/metaopt/optree/pull/156).

Makefile

@@ -140,7 +140,7 @@ pytest test: pytest-install
 	$(PYTHON) -m pytest --version
 	cd tests && $(PYTHON) -X dev -W 'always' -W 'error' -c 'import $(PROJECT_PATH)' && \
 	$(PYTHON) -X dev -W 'always' -W 'error' -c 'import $(PROJECT_PATH)._C; print(f"GLIBCXX_USE_CXX11_ABI={$(PROJECT_PATH)._C.GLIBCXX_USE_CXX11_ABI}")' && \
-	$(PYTHON) -X dev -m pytest --verbose --color=yes --durations=0 --showlocals \
+	$(PYTHON) -X dev -m pytest --verbose --color=yes --durations=10 --showlocals \
 		--cov="$(PROJECT_PATH)" --cov-config=.coveragerc --cov-report=xml --cov-report=term-missing \
 		$(PYTESTOPTS) .
 

include/critical_section.h

@@ -0,0 +1,136 @@
+/*
+Copyright 2022-2024 MetaOPT Team. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+================================================================================
+*/
+
+#pragma once
+
+#include <Python.h>
+
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+
+#ifndef Py_Is
+#define Py_Is(x, y) ((x) == (y))
+#endif
+#ifndef Py_IsNone
+#define Py_IsNone(x) Py_Is((x), Py_None)
+#endif
+#ifndef Py_IsTrue
+#define Py_IsTrue(x) Py_Is((x), Py_True)
+#endif
+#ifndef Py_IsFalse
+#define Py_IsFalse(x) Py_Is((x), Py_False)
+#endif
+
+inline bool Py_IsConstant(PyObject* x) { return Py_IsNone(x) || Py_IsTrue(x) || Py_IsFalse(x); }
+#define Py_IsConstant(x) Py_IsConstant(x)
+
+class scoped_critical_section {
+ public:
+    scoped_critical_section() = delete;
+
+#ifdef Py_GIL_DISABLED
+    explicit scoped_critical_section(const py::handle& handle) : m_ptr{handle.ptr()} {
+        if (m_ptr != nullptr && !Py_IsConstant(m_ptr)) [[likely]] {
+            PyCriticalSection_Begin(&m_critical_section, m_ptr);
+        }
+    }
+
+    ~scoped_critical_section() {
+        if (m_ptr != nullptr && !Py_IsConstant(m_ptr)) [[likely]] {
+            PyCriticalSection_End(&m_critical_section);
+        }
+    }
+#else
+    explicit scoped_critical_section(const py::handle& /*unused*/) {}
+    ~scoped_critical_section() = default;
+#endif
+
+    scoped_critical_section(const scoped_critical_section&) = delete;
+    scoped_critical_section& operator=(const scoped_critical_section&) = delete;
+    scoped_critical_section(scoped_critical_section&&) = delete;
+    scoped_critical_section& operator=(scoped_critical_section&&) = delete;
+
+ private:
+#ifdef Py_GIL_DISABLED
+    PyObject* m_ptr{nullptr};
+    PyCriticalSection m_critical_section{};
+#endif
+};
+
+class scoped_critical_section2 {
+ public:
+    scoped_critical_section2() = delete;
+
+#ifdef Py_GIL_DISABLED
+    explicit scoped_critical_section2(const py::handle& handle1, const py::handle& handle2)
+        : m_ptr1{handle1.ptr()}, m_ptr2{handle2.ptr()} {
+        if (m_ptr1 != nullptr && !Py_IsConstant(m_ptr1)) [[likely]] {
+            if (m_ptr2 != nullptr && !Py_IsConstant(m_ptr2)) [[likely]] {
+                PyCriticalSection2_Begin(&m_critical_section2, m_ptr1, m_ptr2);
+            } else [[unlikely]] {
+                PyCriticalSection_Begin(&m_critical_section, m_ptr1);
+            }
+        } else if (m_ptr2 != nullptr && !Py_IsConstant(m_ptr2)) [[likely]] {
+            PyCriticalSection_Begin(&m_critical_section, m_ptr2);
+        }
+    }
+
+    ~scoped_critical_section2() {
+        if (m_ptr1 != nullptr && !Py_IsConstant(m_ptr1)) [[likely]] {
+            if (m_ptr2 != nullptr && !Py_IsConstant(m_ptr2)) [[likely]] {
+                PyCriticalSection2_End(&m_critical_section2);
+            } else [[unlikely]] {
+                PyCriticalSection_End(&m_critical_section);
+            }
+        } else if (m_ptr2 != nullptr && !Py_IsConstant(m_ptr2)) [[likely]] {
+            PyCriticalSection_End(&m_critical_section);
+        }
+    }
+#else
+    explicit scoped_critical_section2(const py::handle& /*unused*/, const py::handle& /*unused*/) {}
+    ~scoped_critical_section2() = default;
+#endif
+
+    scoped_critical_section2(const scoped_critical_section2&) = delete;
+    scoped_critical_section2& operator=(const scoped_critical_section2&) = delete;
+    scoped_critical_section2(scoped_critical_section2&&) = delete;
+    scoped_critical_section2& operator=(scoped_critical_section2&&) = delete;
+
+ private:
+#ifdef Py_GIL_DISABLED
+    PyObject* m_ptr1{nullptr};
+    PyObject* m_ptr2{nullptr};
+    PyCriticalSection m_critical_section{};
+    PyCriticalSection2 m_critical_section2{};
+#endif
+};
+
+#ifdef Py_GIL_DISABLED
+
+#define EVALUATE_WITH_LOCK_HELD(expression, handle)                                                \
+    (((void)scoped_critical_section{(handle)}), (expression))
+
+#define EVALUATE_WITH_LOCK_HELD2(expression, handle1, handle2)                                     \
+    (((void)scoped_critical_section2{(handle1), (handle2)}), (expression))
+
+#else
+
+#define EVALUATE_WITH_LOCK_HELD(expression, handle) (expression)
+#define EVALUATE_WITH_LOCK_HELD2(expression, handle1, handle2) (expression)
+
+#endif

include/mutex.h

@@ -0,0 +1,75 @@
+/*
+Copyright 2022-2024 MetaOPT Team. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+================================================================================
+*/
+
+#pragma once
+
+#include <mutex>  // std::mutex, std::recursive_mutex, std::lock_guard, std::unique_lock
+
+#include <Python.h>
+
+#ifdef Py_GIL_DISABLED
+
+class pymutex {
+ public:
+    pymutex() = default;
+    ~pymutex() = default;
+
+    pymutex(const pymutex &) = delete;
+    pymutex &operator=(const pymutex &) = delete;
+    pymutex(pymutex &&) = delete;
+    pymutex &operator=(pymutex &&) = delete;
+
+    void lock() { PyMutex_Lock(&mutex); }
+    void unlock() { PyMutex_Unlock(&mutex); }
+
+ private:
+    PyMutex mutex{0};
+};
+
+using mutex = pymutex;
+using recursive_mutex = std::recursive_mutex;
+
+#else
+
+using mutex = std::mutex;
+using recursive_mutex = std::recursive_mutex;
+
+#endif
+
+using scoped_lock_guard = std::lock_guard<mutex>;
+using scoped_recursive_lock_guard = std::lock_guard<recursive_mutex>;
+
+#if (defined(__APPLE__) /* header <shared_mutex> is not available on macOS build target */ &&      \
+     PY_VERSION_HEX < /* Python 3.12.0 */ 0x030C00F0)
+
+#undef HAVE_READ_WRITE_LOCK
+
+using read_write_mutex = mutex;
+using scoped_read_lock_guard = scoped_lock_guard;
+using scoped_write_lock_guard = scoped_lock_guard;
+
+#else
+
+#define HAVE_READ_WRITE_LOCK
+
+#include <shared_mutex>  // std::shared_mutex, std::shared_lock
+
+using read_write_mutex = std::shared_mutex;
+using scoped_read_lock_guard = std::shared_lock<read_write_mutex>;
+using scoped_write_lock_guard = std::unique_lock<read_write_mutex>;
+
+#endif

include/registry.h

@@ -26,6 +26,7 @@ limitations under the License.
 
 #include <pybind11/pybind11.h>
 
+#include "include/mutex.h"
 #include "include/utils.h"
 
 namespace optree {
@@ -137,6 +138,8 @@ class PyTreeTypeRegistry {
                        NamedTypeHash,
                        NamedTypeEq>
         m_named_registrations{};
+
+    static inline read_write_mutex sm_mutex{};
 };
 
 }  // namespace optree