Update python support (add 3.13)

.github/workflows/test.yaml

@@ -8,7 +8,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        py-version: ["3.9", "3.12"]
+        # 3.9: minimum version supported
+        # 3.12 maximum version that supports all dependencies required for examples
+        # 3.13: maximum version supported
+        py-version: ["3.9", "3.12", "3.13"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -19,6 +22,8 @@ jobs:
       - name: Lint
         run: uvx ruff check --fix sematic
       - name: MyPy
+        # Use 3.12 for type checking because 3.13 doesn't have all
+        # example libraries.
         if: matrix.py-version == '3.12'
         run: uv run mypy -p sematic --disable-error-code import-untyped
       - name: Test
@@ -27,21 +32,21 @@ jobs:
           python3 -c "import sys; print(sys.version)"
           pytest
       - name: Check Schemas
-        if: matrix.py-version == '3.12'
+        if: matrix.py-version == '3.13'
         run: git checkout . && make migrate_up_sqlite && make update-schema && git diff --exit-code
       - name: Wheel
-        if: matrix.py-version == '3.12'
+        if: matrix.py-version == '3.13'
         run: |
           make ui
           make uv-wheel
       - uses: actions/upload-artifact@v4
-        if: matrix.py-version == '3.12'
+        if: matrix.py-version == '3.13'
         with:
           name: sematic-wheel
           path: "./dist/sematic-*.whl"
           retention-days: 10
       - uses: actions/upload-artifact@v4
-        if: matrix.py-version == '3.12'
+        if: matrix.py-version == '3.13'
         with:
           name: sematic-wheel-test
           path: "sematic/tests/integration/test_pip_install.sh"

README.md

@@ -7,11 +7,11 @@
 ![PyPI](https://img.shields.io/pypi/v/sematic/0.40.0?style=for-the-badge)
 [![CircleCI](https://img.shields.io/circleci/build/github/sematic-ai/sematic/main?label=CircleCI&style=for-the-badge&token=60d1953bfee5b6bf8201f8e84a10eaa5bf5622fe)](https://app.circleci.com/pipelines/github/sematic-ai/sematic?branch=main&filter=all)
 ![PyPI - License](https://img.shields.io/pypi/l/sematic?style=for-the-badge)
-[![Python 3.8](https://img.shields.io/badge/Python-3.8-blue?style=for-the-badge&logo=none)](https://python.org)
 [![Python 3.9](https://img.shields.io/badge/Python-3.9-blue?style=for-the-badge&logo=none)](https://python.org)
 [![Python 3.10](https://img.shields.io/badge/Python-3.10-blue?style=for-the-badge&logo=none)](https://python.org)
 [![Python 3.11](https://img.shields.io/badge/Python-3.11-blue?style=for-the-badge&logo=none)](https://python.org)
 [![Python 3.12](https://img.shields.io/badge/Python-3.12-blue?style=for-the-badge&logo=none)](https://python.org)
+[![Python 3.13](https://img.shields.io/badge/Python-3.13-blue?style=for-the-badge&logo=none)](https://python.org)
 ![Discord](https://img.shields.io/discord/983789877927747714?label=DISCORD&style=for-the-badge)
 [![Made By Sematic](https://img.shields.io/badge/Made_by-Sematic_🦊-E19632?style=for-the-badge&logo=none)](https://sematic.dev)
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/sematic?style=for-the-badge)

docker/Dockerfile.worker

@@ -1,5 +1,5 @@
 # This is the base image for the worker for cloud execution
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 #FROM nvidia/cuda:11.7.0-base-ubuntu22.04
 
 ARG DEBIAN_FRONTEND=noninteractive
@@ -8,8 +8,8 @@ ENV PATH="/sematic/bin/:${PATH}"
 RUN apt-get update -y
 RUN apt-get install -y software-properties-common
 RUN add-apt-repository -y ppa:deadsnakes/ppa
-RUN apt-get install -y python3.9
+RUN apt-get install -y python3.11
 
 RUN rm /usr/bin/python3
-RUN ln -s /usr/bin/python3.9 /usr/bin/python3
-RUN ln -s /usr/bin/python3.9 /usr/bin/python
+RUN ln -s /usr/bin/python3.11 /usr/bin/python3
+RUN ln -s /usr/bin/python3.11 /usr/bin/python

docs/get-started.md

@@ -6,7 +6,7 @@ Sematic currently supports Linux and MacOS. If you're using Windows, you can
 run Sematic in
 [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/about).
 
-Python versions `3.8` - `3.12` are supported.
+Python versions `3.9` - `3.13` are supported.
 
 ## Installation
 

pyproject.toml

@@ -4,7 +4,7 @@ description = "Sematic ML orchestration tool"
 
 # The formatting of this line must be maintained for version testing.
 version = "0.40.0"
-requires-python = ">=3.9, <3.13"
+requires-python = ">=3.9, <3.14"
 dependencies = [
   # System
   "ipython==8.2.0",
@@ -77,28 +77,32 @@ sematic = "sematic.cli.main:cli"
 [project.optional-dependencies]
 examples = [
   # Examples
-  "snowflake-connector-python==3.12.4",
+  # Most libraries that require python version < 3.13
+  # are marked as such because they do not yet have
+  # full support for python 3.13. Once they do, this restriction
+  # can be removed.
+  "snowflake-connector-python==3.12.4; python_version < '3.13'",
   "pyOpenSSL>=23.0.0",
-  "pyarrow>=12.0.0",
+  "pyarrow>=12.0.0; python_version < '3.13'",
   "python-magic==0.4.27",
-  "torch>=1.13.1",
-  "torchvision>=0.14.1",
-  "pytorch-lightning>=1.6.5",
-  "ray-lightning>=0.3.0",
-  "plotly==5.13.0",
-  "pandas>=1.5.3",
-  "seaborn>=0.12.2",
-  "matplotlib>=3.7.0",
-  "statsmodels>=0.13.5",
-  "scikit-learn>=1.2.1",
-  "numpy>=1.24.0",
-  "xgboost>=1.7.3",
-  "accelerate==0.19.0",
-  "datasets>=2.12.0",
+  "torch>=1.13.1; python_version < '3.13'",
+  "torchvision>=0.14.1; python_version < '3.13'",
+  "pytorch-lightning>=1.6.5; python_version < '3.13'",
+  "ray-lightning>=0.3.0; python_version < '3.13'",
+  "plotly==5.13.0; python_version < '3.13'",
+  "pandas>=1.5.3; python_version < '3.13'",
+  "seaborn>=0.12.2; python_version < '3.13'",
+  "matplotlib>=3.7.0; python_version < '3.13'",
+  "statsmodels>=0.13.5; python_version < '3.13'",
+  "scikit-learn>=1.2.1; python_version < '3.13'",
+  "numpy>=1.24.0; python_version < '3.13'",
+  "xgboost>=1.7.3; python_version < '3.13'",
+  "accelerate==0.19.0; python_version < '3.13'",
+  "datasets>=2.12.0; python_version < '3.13'",
   "huggingface-hub>=0.14.1",
-  "peft>=0.3.0",
-  "transformers>=4.29.2",
-  "gradio>=3.35.2",
+  "peft>=0.3.0; python_version < '3.13'",
+  "transformers>=4.29.2; python_version < '3.13'",
+  "gradio>=3.35.2; python_version < '3.13'",
   "trafilatura>=1.6.0",
   "cohere>=4.9.0",
   "openai>=0.27.8",
@@ -111,18 +115,18 @@ ray = [
   # mean our wheel depends on it, nor does it imply that users
   # will/will not have ray[air]. This is only about what is present
   # in Sematic's dev workspace.
-  "ray[default,air]>=2.3.0",
+  "ray[default,air]>=2.3.0; python_version < '3.13'",
 ]
 
 all = [
-  "ray[default,air]>=2.3.0",
+  "ray[default,air]>=2.3.0; python_version < '3.13'",
 ]
 
 [tool.uv]
 dev-dependencies = [
   "testing-postgresql>=1.3.0",
   "debugpy>=1.6.6",
-  "pandas-stubs>=2.2.2",
+  "pandas-stubs>=2.2.2; python_version < '3.13'",
   "mypy==1.11.1",
   "ruff-lsp==0.0.35",
   "python-lsp-ruff>=2.2.2",

sematic/conftest.py

@@ -0,0 +1,20 @@
+import sys
+from typing import List
+
+
+MAX_PY_VERSIONS = {
+    # These tests depend on libraries that do not yet have
+    # full support for python 3.13. Once they do, they can be added
+    # back into the test suite.
+    "ee/plugins/external_resource/ray/tests/test_checkpoint.py": (3, 13),
+    "ee/plugins/external_resource/ray/tests/test_cluster.py": (3, 13),
+    "types/types/snowflake/tests/test_snowflake_table.py": (3, 13),
+    "types/types/matplotlib/tests/test_figure.py": (3, 13),
+    "types/types/pandas/tests/test_dataframe.py": (3, 13),
+}
+
+collect_ignore: List[str] = []
+
+for path, max_version in MAX_PY_VERSIONS.items():
+    if sys.version_info >= max_version:
+        collect_ignore.append(path)

sematic/examples/add/__main__.yaml

@@ -1,5 +1,5 @@
 version: 1
-base_uri: "sematicai/sematic-worker-base:latest@sha256:bea3926876a3024c33fe08e0a6b2c0377a7eb600d7b3061a3f3f39d711152e3c"
+base_uri: "sematicai/sematic-worker-base:latest@sha256:ce9212acf0ca223c55d55cd91db1a11e1f971adca6817a068f00516264b3cc2b"
 build:
   platform: "linux/amd64"
   requirements: "requirements.txt"

sematic/plugins/building/docker_builder.py

@@ -96,7 +96,7 @@
 RUN python3 -c "import sematic" || \
 ( \
 export PYTHONDONTWRITEBYTECODE=1 && \
-pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && \
+pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && \
 unset PYTHONDONTWRITEBYTECODE \
 )
 

sematic/plugins/building/tests/fixtures/docker/Dockerfile.basic

@@ -8,6 +8,6 @@ RUN python3 -c "from distutils import cmd, util" || ( apt-get update -y && apt-g
 
 RUN which pip || ( export PYTHONDONTWRITEBYTECODE=1 && apt-get update -y && apt-get install -y --no-install-recommends wget && wget --no-verbose -O get-pip.py https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py && unset PYTHONDONTWRITEBYTECODE )
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"

sematic/plugins/building/tests/fixtures/docker/Dockerfile.data

@@ -15,7 +15,7 @@ COPY sematic/plugins/building/tests/fixtures/docker/Dockerfile.requirements sema
 COPY sematic/plugins/building/tests/fixtures/docker/Dockerfile.src sematic/plugins/building/tests/fixtures/docker/Dockerfile.src
 COPY sematic/plugins/building/tests/fixtures/docker/Dockerfile.target sematic/plugins/building/tests/fixtures/docker/Dockerfile.target
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"
 

sematic/plugins/building/tests/fixtures/docker/Dockerfile.full

@@ -18,7 +18,7 @@ COPY sematic/plugins/building/tests/fixtures/nested/third_level/third_level.txt
 COPY sematic/plugins/building/tests/fixtures/requirements.txt requirements.txt
 RUN pip install --no-cache-dir --ignore-installed --root-user-action=ignore -r requirements.txt
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"
 

sematic/plugins/building/tests/fixtures/docker/Dockerfile.requirements

@@ -11,7 +11,7 @@ RUN which pip || ( export PYTHONDONTWRITEBYTECODE=1 && apt-get update -y && apt-
 COPY sematic/plugins/building/tests/fixtures/requirements.txt requirements.txt
 RUN pip install --no-cache-dir --ignore-installed --root-user-action=ignore -r requirements.txt
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"
 

sematic/plugins/building/tests/fixtures/docker/Dockerfile.src

@@ -8,7 +8,7 @@ RUN python3 -c "from distutils import cmd, util" || ( apt-get update -y && apt-g
 
 RUN which pip || ( export PYTHONDONTWRITEBYTECODE=1 && apt-get update -y && apt-get install -y --no-install-recommends wget && wget --no-verbose -O get-pip.py https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py && unset PYTHONDONTWRITEBYTECODE )
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"
 

sematic/plugins/building/tests/fixtures/docker/Dockerfile.target

@@ -8,7 +8,7 @@ RUN python3 -c "from distutils import cmd, util" || ( apt-get update -y && apt-g
 
 RUN which pip || ( export PYTHONDONTWRITEBYTECODE=1 && apt-get update -y && apt-get install -y --no-install-recommends wget && wget --no-verbose -O get-pip.py https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py && unset PYTHONDONTWRITEBYTECODE )
 
-RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic && unset PYTHONDONTWRITEBYTECODE )
+RUN python3 -c "import sematic" || ( export PYTHONDONTWRITEBYTECODE=1 && pip install --no-cache-dir --ignore-installed --root-user-action=ignore sematic cffi && unset PYTHONDONTWRITEBYTECODE )
 
 ENV PATH="/sematic/bin/:$PATH"
 

sematic/types/types/snowflake/snowflake_table.py

@@ -9,7 +9,7 @@
     # Third-party
     import pandas
     import pyarrow  # type: ignore  # noqa: F401
-    import snowflake.connector
+    import snowflake.connector  # type: ignore
 except ImportError as e:
     print(
         "You are attempting to use SnowflakeTable which requires the following dependencies:"  # noqa: E501

sematic/utils/git.py

@@ -30,7 +30,7 @@ def get_git_info(object_: Any) -> Optional[GitInfo]:
         # Third-party
         import git  # type: ignore
     except ImportError as e:
-        logger.warn("Could not get git information", exc_info=e)
+        logger.warning("Could not get git information", exc_info=e)
         return None
 
     # try to search the path ancestors of the given object's source code for a git repo