Update python support (add 3.13) (#1143)

Add support for python 3.13. Not all of the dependencies we have for
examples have support for 3.13 on all platforms, so some tests need to
be skipped in that case. Also update the Sematic worker image to use
ubuntu 24.04 and python 3.12. Leaving Sematic server at 3.12, because
the Sematic EE image needs Ray, which doesn't have full support for 3.13
yet.

Testing
-------
In addition to the new CI unit tests, ran the `add` example in the cloud
with a newly pushed Sematic worker base image.

.github/workflows/test.yaml

@@ -8,7 +8,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        py-version: ["3.9", "3.12"]
+        # 3.9: minimum version supported
+        # 3.12 maximum version that supports all dependencies required for examples
+        # 3.13: maximum version supported
+        py-version: ["3.9", "3.12", "3.13"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -19,6 +22,8 @@ jobs:
       - name: Lint
         run: uvx ruff check --fix sematic
       - name: MyPy
+        # Use 3.12 for type checking because 3.13 doesn't have all
+        # example libraries.
         if: matrix.py-version == '3.12'
         run: uv run mypy -p sematic --disable-error-code import-untyped
       - name: Test
@@ -27,21 +32,21 @@ jobs:
           python3 -c "import sys; print(sys.version)"
           pytest
       - name: Check Schemas
-        if: matrix.py-version == '3.12'
+        if: matrix.py-version == '3.13'
         run: git checkout . && make migrate_up_sqlite && make update-schema && git diff --exit-code
       - name: Wheel
-        if: matrix.py-version == '3.12'
+        if: matrix.py-version == '3.13'
         run: |
           make ui
           make uv-wheel
       - uses: actions/upload-artifact@v4
-        if: matrix.py-version == '3.12'
+        if: matrix.py-version == '3.13'
         with:
           name: sematic-wheel
           path: "./dist/sematic-*.whl"
           retention-days: 10
       - uses: actions/upload-artifact@v4
-        if: matrix.py-version == '3.12'
+        if: matrix.py-version == '3.13'
         with:
           name: sematic-wheel-test
           path: "sematic/tests/integration/test_pip_install.sh"

README.md

@@ -7,11 +7,11 @@
 ![PyPI](https://img.shields.io/pypi/v/sematic/0.40.0?style=for-the-badge)
 [![CircleCI](https://img.shields.io/circleci/build/github/sematic-ai/sematic/main?label=CircleCI&style=for-the-badge&token=60d1953bfee5b6bf8201f8e84a10eaa5bf5622fe)](https://app.circleci.com/pipelines/github/sematic-ai/sematic?branch=main&filter=all)
 ![PyPI - License](https://img.shields.io/pypi/l/sematic?style=for-the-badge)
-[![Python 3.8](https://img.shields.io/badge/Python-3.8-blue?style=for-the-badge&logo=none)](https://python.org)
 [![Python 3.9](https://img.shields.io/badge/Python-3.9-blue?style=for-the-badge&logo=none)](https://python.org)
 [![Python 3.10](https://img.shields.io/badge/Python-3.10-blue?style=for-the-badge&logo=none)](https://python.org)
 [![Python 3.11](https://img.shields.io/badge/Python-3.11-blue?style=for-the-badge&logo=none)](https://python.org)
 [![Python 3.12](https://img.shields.io/badge/Python-3.12-blue?style=for-the-badge&logo=none)](https://python.org)
+[![Python 3.13](https://img.shields.io/badge/Python-3.13-blue?style=for-the-badge&logo=none)](https://python.org)
 ![Discord](https://img.shields.io/discord/983789877927747714?label=DISCORD&style=for-the-badge)
 [![Made By Sematic](https://img.shields.io/badge/Made_by-Sematic_🦊-E19632?style=for-the-badge&logo=none)](https://sematic.dev)
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/sematic?style=for-the-badge)

docker/Dockerfile.worker

@@ -1,5 +1,5 @@
 # This is the base image for the worker for cloud execution
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 #FROM nvidia/cuda:11.7.0-base-ubuntu22.04
 
 ARG DEBIAN_FRONTEND=noninteractive
@@ -8,8 +8,8 @@ ENV PATH="/sematic/bin/:${PATH}"
 RUN apt-get update -y
 RUN apt-get install -y software-properties-common
 RUN add-apt-repository -y ppa:deadsnakes/ppa
-RUN apt-get install -y python3.9
+RUN apt-get install -y python3.11
 
 RUN rm /usr/bin/python3
-RUN ln -s /usr/bin/python3.9 /usr/bin/python3
-RUN ln -s /usr/bin/python3.9 /usr/bin/python
+RUN ln -s /usr/bin/python3.11 /usr/bin/python3
+RUN ln -s /usr/bin/python3.11 /usr/bin/python

docs/get-started.md

@@ -6,7 +6,7 @@ Sematic currently supports Linux and MacOS. If you're using Windows, you can
 run Sematic in
 [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/about).
 
-Python versions `3.8` - `3.12` are supported.
+Python versions `3.9` - `3.13` are supported.
 
 ## Installation
 

pyproject.toml

@@ -4,7 +4,7 @@ description = "Sematic ML orchestration tool"
 
 # The formatting of this line must be maintained for version testing.
 version = "0.40.0"
-requires-python = ">=3.9, <3.13"
+requires-python = ">=3.9, <3.14"
 dependencies = [
   # System
   "ipython==8.2.0",
@@ -77,28 +77,32 @@ sematic = "sematic.cli.main:cli"
 [project.optional-dependencies]
 examples = [
   # Examples
-  "snowflake-connector-python==3.12.4",
+  # Most libraries that require python version < 3.13
+  # are marked as such because they do not yet have
+  # full support for python 3.13. Once they do, this restriction
+  # can be removed.
+  "snowflake-connector-python==3.12.4; python_version < '3.13'",
   "pyOpenSSL>=23.0.0",
-  "pyarrow>=12.0.0",
+  "pyarrow>=12.0.0; python_version < '3.13'",
   "python-magic==0.4.27",
-  "torch>=1.13.1",
-  "torchvision>=0.14.1",
-  "pytorch-lightning>=1.6.5",
-  "ray-lightning>=0.3.0",
-  "plotly==5.13.0",
-  "pandas>=1.5.3",
-  "seaborn>=0.12.2",
-  "matplotlib>=3.7.0",
-  "statsmodels>=0.13.5",
-  "scikit-learn>=1.2.1",
-  "numpy>=1.24.0",
-  "xgboost>=1.7.3",
-  "accelerate==0.19.0",
-  "datasets>=2.12.0",
+  "torch>=1.13.1; python_version < '3.13'",
+  "torchvision>=0.14.1; python_version < '3.13'",
+  "pytorch-lightning>=1.6.5; python_version < '3.13'",
+  "ray-lightning>=0.3.0; python_version < '3.13'",
+  "plotly==5.13.0; python_version < '3.13'",
+  "pandas>=1.5.3; python_version < '3.13'",
+  "seaborn>=0.12.2; python_version < '3.13'",
+  "matplotlib>=3.7.0; python_version < '3.13'",
+  "statsmodels>=0.13.5; python_version < '3.13'",
+  "scikit-learn>=1.2.1; python_version < '3.13'",
+  "numpy>=1.24.0; python_version < '3.13'",
+  "xgboost>=1.7.3; python_version < '3.13'",
+  "accelerate==0.19.0; python_version < '3.13'",
+  "datasets>=2.12.0; python_version < '3.13'",
   "huggingface-hub>=0.14.1",
-  "peft>=0.3.0",
-  "transformers>=4.29.2",
-  "gradio>=3.35.2",
+  "peft>=0.3.0; python_version < '3.13'",
+  "transformers>=4.29.2; python_version < '3.13'",
+  "gradio>=3.35.2; python_version < '3.13'",
   "trafilatura>=1.6.0",
   "cohere>=4.9.0",
   "openai>=0.27.8",
@@ -111,18 +115,18 @@ ray = [
   # mean our wheel depends on it, nor does it imply that users
   # will/will not have ray[air]. This is only about what is present
   # in Sematic's dev workspace.
-  "ray[default,air]>=2.3.0",
+  "ray[default,air]>=2.3.0; python_version < '3.13'",
 ]
 
 all = [
-  "ray[default,air]>=2.3.0",
+  "ray[default,air]>=2.3.0; python_version < '3.13'",
 ]
 
 [tool.uv]
 dev-dependencies = [
   "testing-postgresql>=1.3.0",
   "debugpy>=1.6.6",
-  "pandas-stubs>=2.2.2",
+  "pandas-stubs>=2.2.2; python_version < '3.13'",
   "mypy==1.11.1",
   "ruff-lsp==0.0.35",
   "python-lsp-ruff>=2.2.2",

sematic/conftest.py

@@ -0,0 +1,20 @@
+import sys
+from typing import List
+
+
+MAX_PY_VERSIONS = {
+    # These tests depend on libraries that do not yet have
+    # full support for python 3.13. Once they do, they can be added
+    # back into the test suite.
+    "ee/plugins/external_resource/ray/tests/test_checkpoint.py": (3, 13),
+    "ee/plugins/external_resource/ray/tests/test_cluster.py": (3, 13),
+    "types/types/snowflake/tests/test_snowflake_table.py": (3, 13),
+    "types/types/matplotlib/tests/test_figure.py": (3, 13),
+    "types/types/pandas/tests/test_dataframe.py": (3, 13),
+}
+
+collect_ignore: List[str] = []
+
+for path, max_version in MAX_PY_VERSIONS.items():
+    if sys.version_info >= max_version:
+        collect_ignore.append(path)

sematic/examples/add/__main__.yaml

@@ -1,5 +1,5 @@
 version: 1
-base_uri: "sematicai/sematic-worker-base:latest@sha256:bea3926876a3024c33fe08e0a6b2c0377a7eb600d7b3061a3f3f39d711152e3c"
+base_uri: "sematicai/sematic-worker-base:latest@sha256:ce9212acf0ca223c55d55cd91db1a11e1f971adca6817a068f00516264b3cc2b"
 build:
   platform: "linux/amd64"
   requirements: "requirements.txt"

sematic/plugins/building/docker_builder.py

@@ -96,7 +96,7 @@
 RUN python3 -c "import sematic" || \
 ( \
 export PYTHONDONTWRITEBYTECODE=1 && \
-pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && \
+pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && \
 unset PYTHONDONTWRITEBYTECODE \
 )
 

sematic/plugins/building/tests/fixtures/docker/Dockerfile.basic

@@ -8,6 +8,6 @@ RUN python3 -c "from distutils import cmd, util" || ( apt-get update -y && apt-g
 
 RUN which pip || ( export PYTHONDONTWRITEBYTECODE=1 && apt-get update -y && apt-get install -y --no-install-recommends wget && wget --no-verbose -O get-pip.py https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py && unset PYTHONDONTWRITEBYTECODE )
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"

sematic/plugins/building/tests/fixtures/docker/Dockerfile.data

@@ -15,7 +15,7 @@ COPY sematic/plugins/building/tests/fixtures/docker/Dockerfile.requirements sema
 COPY sematic/plugins/building/tests/fixtures/docker/Dockerfile.src sematic/plugins/building/tests/fixtures/docker/Dockerfile.src
 COPY sematic/plugins/building/tests/fixtures/docker/Dockerfile.target sematic/plugins/building/tests/fixtures/docker/Dockerfile.target
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"
 

sematic/plugins/building/tests/fixtures/docker/Dockerfile.full

@@ -18,7 +18,7 @@ COPY sematic/plugins/building/tests/fixtures/nested/third_level/third_level.txt
 COPY sematic/plugins/building/tests/fixtures/requirements.txt requirements.txt
 RUN pip install --no-cache-dir --ignore-installed --root-user-action=ignore -r requirements.txt
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"
 

sematic/plugins/building/tests/fixtures/docker/Dockerfile.requirements

@@ -11,7 +11,7 @@ RUN which pip || ( export PYTHONDONTWRITEBYTECODE=1 && apt-get update -y && apt-
 COPY sematic/plugins/building/tests/fixtures/requirements.txt requirements.txt
 RUN pip install --no-cache-dir --ignore-installed --root-user-action=ignore -r requirements.txt
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"
 

sematic/plugins/building/tests/fixtures/docker/Dockerfile.src

@@ -8,7 +8,7 @@ RUN python3 -c "from distutils import cmd, util" || ( apt-get update -y && apt-g
 
 RUN which pip || ( export PYTHONDONTWRITEBYTECODE=1 && apt-get update -y && apt-get install -y --no-install-recommends wget && wget --no-verbose -O get-pip.py https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py && unset PYTHONDONTWRITEBYTECODE )
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"
 

sematic/plugins/building/tests/fixtures/docker/Dockerfile.target

@@ -8,7 +8,7 @@ RUN python3 -c "from distutils import cmd, util" || ( apt-get update -y && apt-g
 
 RUN which pip || ( export PYTHONDONTWRITEBYTECODE=1 && apt-get update -y && apt-get install -y --no-install-recommends wget && wget --no-verbose -O get-pip.py https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py && unset PYTHONDONTWRITEBYTECODE )
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"
 

sematic/types/types/snowflake/snowflake_table.py

@@ -9,7 +9,7 @@
     # Third-party
     import pandas
     import pyarrow  # type: ignore  # noqa: F401
-    import snowflake.connector
+    import snowflake.connector  # type: ignore
 except ImportError as e:
     print(
         "You are attempting to use SnowflakeTable which requires the following dependencies:"  # noqa: E501

sematic/utils/git.py

@@ -30,7 +30,7 @@ def get_git_info(object_: Any) -> Optional[GitInfo]:
         # Third-party
         import git  # type: ignore
     except ImportError as e:
-        logger.warn("Could not get git information", exc_info=e)
+        logger.warning("Could not get git information", exc_info=e)
         return None
 
     # try to search the path ancestors of the given object's source code for a git repo