Merge branch 'main' into feature/methylVI-model

.github/workflows/build.yml

@@ -19,7 +19,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           cache: "pip"
           cache-dependency-path: "**/pyproject.toml"
 

.github/workflows/build_image_base.yaml

@@ -38,4 +38,4 @@ jobs:
           cache-from: type=registry,ref=ghcr.io/scverse/scvi-tools:buildcache
           cache-to: type=inline,ref=ghcr.io/scverse/scvi-tools:buildcache
           target: base
-          tags: ghcr.io/scverse/scvi-tools:py3.11-cu12-base
+          tags: ghcr.io/scverse/scvi-tools:py3.12-cu12-base

.github/workflows/build_image_latest.yaml

@@ -60,6 +60,6 @@ jobs:
           cache-from: type=registry,ref=ghcr.io/scverse/scvi-tools:buildcache
           cache-to: type=inline,ref=ghcr.io/scverse/scvi-tools:buildcache
           target: build
-          tags: ghcr.io/scverse/scvi-tools:py3.11-cu12-${{ steps.build.outputs.version }}-${{ steps.build.outputs.dependencies }}
+          tags: ghcr.io/scverse/scvi-tools:py3.12-cu12-${{ steps.build.outputs.version }}-${{ steps.build.outputs.dependencies }}
           build-args: |
             DEPENDENCIES=${{ matrix.dependencies }}

.github/workflows/release.yml

@@ -112,7 +112,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - run: pip install build
 
@@ -158,6 +158,6 @@ jobs:
           cache-from: type=registry,ref=ghcr.io/scverse/scvi-tools:buildcache
           cache-to: type=inline,ref=ghcr.io/scverse/scvi-tools:buildcache
           target: build
-          tags: ghcr.io/scverse/scvi-tools:py3.11-cu12-${{ inputs.tag }}-${{ matrix.dependencies }},ghcr.io/scverse/scvi-tools:py3.11-cu12-stable-${{ matrix.dependencies }}
+          tags: ghcr.io/scverse/scvi-tools:py3.12-cu12-${{ inputs.tag }}-${{ matrix.dependencies }},ghcr.io/scverse/scvi-tools:py3.12-cu12-stable-${{ matrix.dependencies }}
           build-args: |
             DEPENDENCIES=${{ matrix.dependencies }}

.github/workflows/test_linux.yml

@@ -24,7 +24,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python: ["3.9", "3.10", "3.11"]
+        python: ["3.10", "3.11", "3.12"]
 
     name: integration
 

.github/workflows/test_linux_cuda.yml

@@ -30,7 +30,7 @@ jobs:
         shell: bash -e {0} # -e to fail on error
 
     container:
-      image: ghcr.io/scverse/scvi-tools:py3.11-cu12-base
+      image: ghcr.io/scverse/scvi-tools:py3.12-cu12-base
       options: --user root --gpus all
 
     name: integration

.github/workflows/test_linux_private.yml

@@ -34,7 +34,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python: ["3.11"]
+        python: ["3.12"]
 
     permissions:
       id-token: write

.github/workflows/test_linux_resolution.yml

@@ -33,13 +33,8 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python: ["3.9", "3.10", "3.11", "3.12"]
-        install-flags:
-          [
-            "--prerelease if-necessary-or-explicit",
-            "--resolution lowest-direct",
-            "--resolution lowest",
-          ]
+        python: ["3.10", "3.11", "3.12"]
+        install-flags: ["--prerelease if-necessary-or-explicit"]
 
     name: integration
 

.github/workflows/test_macos.yml

@@ -34,7 +34,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-14]
-        python: ["3.9", "3.10", "3.11"]
+        python: ["3.10", "3.11", "3.12"]
 
     name: integration
 

.github/workflows/test_windows.yml

@@ -34,7 +34,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [windows-latest]
-        python: ["3.9", "3.10", "3.11"]
+        python: ["3.10", "3.11", "3.12"]
 
     name: integration
 

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
           )$
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.41.0
+    rev: v0.42.0
     hooks:
       - id: markdownlint-fix
         exclude: |
@@ -41,7 +41,7 @@ repos:
           )$
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.2
+    rev: v0.6.8
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

CHANGELOG.md

@@ -4,12 +4,13 @@ Starting from version 0.20.1, this format is based on [Keep a Changelog], and th
 to [Semantic Versioning]. Full commit history is available in the
 [commit logs](https://github.com/scverse/scvi-tools/commits/).
 
-## Version 1.2 (unreleased)
+## Version 1.2
 
-### 1.2.0 (unreleased)
+### 1.2.0 (2024-09-26)
 
 #### Added
 
+- Add support for Python 3.12 {pr}`2966`.
 - Add support for categorial covariates in scArches in `scvi.model.archesmixin` {pr}`2936`.
 - Add assertion error in cellAssign for checking duplicates in celltype markers {pr}`2951`.
 - Add `scvi.external.poissonvi.get_region_factors` {pr}`2940`.

Dockerfile

@@ -1,5 +1,5 @@
-FROM nvidia/cuda:12.5.0-runtime-ubuntu22.04
-FROM python:3.11 AS base
+FROM nvidia/cuda:12.4.0-runtime-ubuntu22.04
+FROM python:3.12 AS base
 
 RUN pip install --no-cache-dir uv
 
@@ -9,7 +9,7 @@ CMD ["/bin/bash"]
 
 FROM base AS build
 
-ENV SCVI_PATH="/usr/local/lib/python3.11/site-packages/scvi-tools"
+ENV SCVI_PATH="/usr/local/lib/python3.12/site-packages/scvi-tools"
 
 COPY . ${SCVI_PATH}
 

docs/conf.py

@@ -5,9 +5,12 @@
 import subprocess
 import sys
 from pathlib import Path
-from typing import Any
 from importlib.metadata import metadata
 from datetime import datetime
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
 
 HERE = Path(__file__).parent
 sys.path[:0] = [str(HERE.parent), str(HERE / "extensions")]

docs/extensions/typed_returns.py

@@ -3,10 +3,13 @@
 from __future__ import annotations
 
 import re
-from collections.abc import Generator, Iterable
 
-from sphinx.application import Sphinx
 from sphinx.ext.napoleon import NumpyDocstring
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+    from collections.abc import Generator, Iterable
 
 
 def _process_return(lines: Iterable[str]) -> Generator[str, None, None]:

docs/installation.md

@@ -23,14 +23,14 @@ Don't know how to get started with virtual environments or `conda`/`pip`? Check
 ### Virtual environment
 
 A virtual environment can be created with either `conda` or `venv`. We recommend using `conda`. We
-currently support Python 3.9 - 3.11.
+currently support Python 3.10 - 3.12.
 
 For `conda`, we recommend using the [Miniforge](https://github.com/conda-forge/miniforge)
 distribution, which is generally faster than the official distribution and comes with conda-forge
 as the default channel (where scvi-tools is hosted).
 
 ```bash
-conda create -n scvi-env python=3.11  # any python 3.9 to 3.11
+conda create -n scvi-env python=3.12  # any python 3.10 to 3.12
 conda activate scvi-env
 ```
 

docs/user_guide/background/counterfactual_prediction.md

@@ -1,5 +1,61 @@
 # Counterfactual prediction
 
-:::{note}
-This page is under construction.
+Once we have trained a model to predict a variable of interest or a generative model to learn the data distribution, we are often interested in making predictions for new samples. However, predictions over test samples may not reveal exactly what the model has learned about how the input features relate to the target variable of interest. For example, we may want to answer the question: How would the model predict the expression levels of gene Y in cell Z if gene X is knocked out? Even if we do not have an data point corresponding to this scenario, we can instead perturb the input to see what the model reports.
+
+:::{warning}
+We are using the term "counterfactual prediction" here rather loosely. In particular, we are not following the rigorous definition of counterfactual prediction in the causality literature[^ref1]. While closely related in spirit, we are making counterfactual queries with statistical models to gain some insight into what the model has learned about the data distribution.
+:::
+
+:::{figure} figures/counterfactual_cartoon.svg
+:align: center
+:alt: Cartoon of the counterfactual prediction task across two conditions.
+:class: img-fluid
+
+Cartoon of the counterfactual prediction task across two conditions. This counterfactual prediction can be thought of as an interpolation of nearby points in the feature space originating from condition B.
 :::
+
+## Preliminaries
+
+Suppose we have a trained model $f_\theta$ that takes in a data point $x$ (e.g., gene expression counts) and a condition $c$ (e.g., treatment group) and returns a prediction $\hat{y}$.
+Each data point takes the form of a tuple $(x,c) \in \mathcal{D}$.
+We can define a *counterfactual query* as a pair $(x,c')$ where $c' \neq c$,
+and the respective model output as the *counterfactual prediction*, $\hat{y}' = f_\theta(x,c')$.
+
+We separate $c$ here out from $x$ to make the counterfactual portion of the query explicit, but it can be thought of as another dimension of $x$.
+
+## In-distribution vs. out-of-distribution
+
+Since we are working with statistical models rather than causal models, we have to be careful when we can rely on counterfactual predictions. At a high level, if we assume the true function relating the features to the target is smooth, we can trust counterfactual predictions for queries that are similar to points in the training data.
+
+Say we have a counterfactual query $(x,c')$, and we have data points in the training set $(x',c')$ (i.e., $\|x - x'\|$ is small).
+If our model predicts the $y$ for $(x', c')$ well,
+we can reasonably trust the counterfactual prediction for $(x,c')$.
+Otherwise, if $(x,c')$ is very different from any point in the training data
+with condition $c'$, we cannot make any guarantees about the accuracy of the counterfactual prediction.
+Dimensionality reduction techniques or harmonization methods may help create more overlap between the features $x$ across the conditions, setting the stage for more reliable counterfactual predictions.
+
+## Applications
+
+The most direct application of counterfactual prediction in scvi-tools can be found in the `transform_batch` kwarg of the {func}`~scvi.model.SCVI.get_normalized_expression` function. In this case, we can pass in a counterfactual batch label to get a prediction of what the normalized expression would be for a cell if it were a member of that batch. This can be useful if one wants to compare cells across different batches in the gene space.
+
+The described approach to counterfactual prediction has also been used in a variety of applications, including:
+- characterizing cell-type-specific sample-level effects [^ref2]
+- predicting chemical perturbation responses in different cell types [^ref2][^ref3]
+- predicting infection/perturbation responses across species [^ref4]
+
+For more details on how counterfactual prediction is used in another method implemented in scvi-tools, see the {doc}`/user_guide/models/mrvi`.
+
+[^ref1]:
+    Judea Pearl. Causality. Cambridge university press, 2009.
+[^ref2]:
+     Pierre Boyeau, Justin Hong, Adam Gayoso, Martin Kim, Jose L McFaline-Figueroa, Michael Jordan, Elham Azizi, Can Ergen, Nir Yosef (2024),
+    _Deep generative modeling of sample-level heterogeneity in single-cell genomics_,
+    [bioRxiv](https://doi.org/10.1101/2022.10.04.510898).
+[^ref3]:
+    Mohammad Lotfollahi, Anna Klimovskaia Susmelj, Carlo De Donno, Leon Hetzel, Yuge Ji, Ignacio L Ibarra, Sanjay R Srivatsan, Mohsen Naghipourfar, Riza M Daza, Beth Martin, Jay Shendure, Jose L McFaline‐Figueroa, Pierre Boyeau, F Alexander Wolf, Nafissa Yakubova, Stephan Günnemann, Cole Trapnell, David Lopez‐Paz, Fabian J Theis (2023),
+    _Predicting cellular responses to complex perturbations in high‐throughput screens_,
+    [Molecular Systems Biology](https://doi.org/10.15252/msb.202211517).
+[^ref4]:
+    Mohammad Lotfollahi, F Alexander Wolf, Fabian J Theis (2019),
+    _scGen predicts single-cell perturbation responses_,
+    [Nature Methods](https://doi.org/10.1038/s41592-019-0494-8).

pyproject.toml

@@ -4,27 +4,28 @@ requires = ["hatchling"]
 
 [project]
 name = "scvi-tools"
-version = "1.1.6"
+version = "1.2.0"
 description = "Deep probabilistic analysis of single-cell omics data."
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 license = {file = "LICENSE"}
 authors = [
     {name = "The scvi-tools development team"},
 ]
 maintainers = [
-    {name = "The scvi-tools development team", email = "[email protected]"},
+    {name = "The scvi-tools development team", email = "[email protected]"},
 ]
+
 urls.Documentation = "https://scvi-tools.org"
 urls.Source = "https://github.com/scverse/scvi-tools"
 urls.Home-page = "https://scvi-tools.org"
 classifiers = [
   "Development Status :: 4 - Beta",
   "Intended Audience :: Science/Research",
   "Natural Language :: English",
-  "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Operating System :: MacOS :: MacOS X",
   "Operating System :: Microsoft :: Windows",
   "Operating System :: POSIX :: Linux",
@@ -55,7 +56,6 @@ dependencies = [
     "xarray>=2023.2.0",
 ]
 
-
 [project.optional-dependencies]
 tests = ["pytest", "pytest-pretty", "coverage", "scvi-tools[optional]"]
 editing = ["jupyter", "pre-commit"]
@@ -99,7 +99,6 @@ optional = [
 ]
 tutorials = [
     "cell2location",
-    "gdown",
     "jupyter",
     "leidenalg",
     "muon",
@@ -109,6 +108,7 @@ tutorials = [
     "igraph",
     "scikit-misc",
     "scrublet",
+    "scib-metrics",
     "scvi-tools[optional]",
     "squidpy",
 ]
@@ -137,7 +137,7 @@ markers = [
 src = ["src"]
 line-length = 99
 indent-width = 4
-target-version = "py39"
+target-version = "py312"
 
 # Exclude a variety of commonly ignored directories.
 exclude = [
@@ -177,6 +177,10 @@ select = [
     "BLE",  # flake8-blind-except
     "UP",  # pyupgrade
     "RUF100",  # Report unused noqa directives
+    "PT", # pytest style
+    "NPY", # numpy formatting
+    "TCH", # flake8-type-checking
+    "FA", # flake8-future-annotations
 ]
 ignore = [
     # allow I, O, l as variable names -> I is the identity matrix
@@ -189,17 +193,16 @@ ignore = [
     "D107",
     # Errors from function calls in argument defaults. These are fine when the result is immutable.
     "B008",
-    # __magic__ methods are are often self-explanatory, allow missing docstrings
-    "D105",
     # first line should end with a period [Bug: doesn't work with single-line docstrings]
     "D400",
     # First line should be in imperative mood; try rephrasing
     "D401",
-    ## Disable one in each pair of mutually incompatible rules
-    # We don’t want a blank line before a class docstring
-    "D203",
     # We want docstrings to start immediately after the opening triple quote
     "D213",
+    # Raising ValueError is sufficient in tests.
+    "PT011",
+    # We support np.random functions.
+    "NPY002"
 ]
 
 [tool.ruff.lint.pydocstyle]
@@ -212,6 +215,7 @@ convention = "numpy"
 "src/scvi/__init__.py" = ["I"]
 
 [tool.ruff.format]
+docstring-code-format = true
 # Like Black, use double quotes for strings.
 quote-style = "double"
 
@@ -226,3 +230,7 @@ line-ending = "auto"
 
 [tool.jupytext]
 formats = "ipynb,md"
+
+[tool.ruff.lint.flake8-type-checking]
+exempt-modules = []
+strict = true