diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 653769fbca..27e95f4656 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -62,6 +62,13 @@ jobs: pip install --editable .[evaluate] python -c "import openproblems" + - name: Download docker images + run: | + for image in $(find ./docker -mindepth 1 -type d -exec basename {} \;); do + docker pull singlecellopenproblems/${image} & + done + wait + - name: Update Docker images if: | !( diff --git a/docker/openproblems-python-batch-integration/Dockerfile b/docker/openproblems-python-batch-integration/Dockerfile new file mode 100644 index 0000000000..697905d79c --- /dev/null +++ b/docker/openproblems-python-batch-integration/Dockerfile @@ -0,0 +1,24 @@ +FROM singlecellopenproblems/openproblems-r-base:latest + +USER root +WORKDIR / + +ARG NB_USER="sagemaker-user" +ARG NB_UID="1000" +ARG NB_GID="100" + +RUN sed -i '$ d' /etc/apt/sources.list +RUN \ +apt-get update --allow-releaseinfo-change && \ +apt-get -y install --no-install-recommends gcc git python3-llvmlite && \ +apt-get autoremove -y && \ +rm -rf /var/lib/apt/lists/* + +# Install Python packages +COPY ./docker/openproblems-python-batch-integration/requirements.txt ./requirements.txt +RUN pip install --no-cache-dir -r requirements.txt +# force reinstall annoy addresses https://github.com/spotify/annoy/issues/513 +RUN pip install --no-cache-dir --force annoy==1.17.0 + +USER $NB_UID +WORKDIR /home/$NB_USER diff --git a/docker/openproblems-python-batch-integration/README.md b/docker/openproblems-python-batch-integration/README.md new file mode 100644 index 0000000000..02a18e1c20 --- /dev/null +++ b/docker/openproblems-python-batch-integration/README.md @@ -0,0 +1,15 @@ +# openproblems-python-extras Docker image + +Base image: singlecellopenproblems/openproblems-r-base + +OS: Debian Stretch + +Python: 3.8 + +Python packages: + +* scIB +* mnnpy +* scanorama +* bbknn +* scVI diff --git a/docker/openproblems-python-batch-integration/requirements.txt b/docker/openproblems-python-batch-integration/requirements.txt new file mode 100644 index 0000000000..9c92302df9 --- /dev/null +++ b/docker/openproblems-python-batch-integration/requirements.txt @@ -0,0 +1,6 @@ +annoy==1.17.1 +bbknn==1.5.* +git+https://github.com/scottgigante-immunai/mnnpy@eb4c551 # branch: patch-2 +git+https://github.com/theislab/scib@77ab015 +scanorama==1.7.0 +scvi-tools~=0.16 # pinned in #313 diff --git a/docker/openproblems-python-bedtools/Dockerfile b/docker/openproblems-python-bedtools/Dockerfile deleted file mode 100644 index d2ad393378..0000000000 --- a/docker/openproblems-python-bedtools/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM singlecellopenproblems/openproblems:latest - -ARG NB_USER="sagemaker-user" -ARG NB_UID="1000" -ARG NB_GID="100" - -USER root -WORKDIR / - -# Install pybedtools dependency -ARG BUILD_PACKAGES="" -ARG PACKAGE_VERSION=2.27.1 -RUN apt-get update && \ - apt-get install --yes git openssl build-essential zlib1g-dev && \ - cd /tmp && \ - git clone https://github.com/arq5x/bedtools2.git && \ - cd bedtools2 && \ - git checkout v$PACKAGE_VERSION && \ - make && \ - mv bin/* /usr/local/bin && \ - cd / - -# install dependencies and openproblems -COPY ./docker/openproblems-python-bedtools/requirements.txt ./requirements.txt -RUN pip install --no-cache-dir -r requirements.txt - -USER $NB_UID -WORKDIR /home/$NB_USER diff --git a/docker/openproblems-python-bedtools/README.md b/docker/openproblems-python-bedtools/README.md deleted file mode 100644 index d7b9935395..0000000000 --- a/docker/openproblems-python-bedtools/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# openproblems-python-extras Docker image - -Base image: singlecellopenproblems/openproblems - -OS: Debian Stretch - -Python: 3.8 - -Python packages: - -* pybedtools -* pyensembl diff --git a/docker/openproblems-python-bedtools/requirements.txt b/docker/openproblems-python-bedtools/requirements.txt deleted file mode 100644 index 5f308af4ed..0000000000 --- a/docker/openproblems-python-bedtools/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pybedtools==0.9.* -pyensembl==2.0.* diff --git a/docker/openproblems-python-extras/requirements.txt b/docker/openproblems-python-extras/requirements.txt index 7d6aee63d5..c51a5d771a 100644 --- a/docker/openproblems-python-extras/requirements.txt +++ b/docker/openproblems-python-extras/requirements.txt @@ -1,7 +1,17 @@ cmake==3.24.1.1 +git+https://github.com/BayraktarLab/cell2location.git@7e7aa231cc61ff460da14402fa3b9a1fa3ec69ac +git+https://github.com/czbiohub/molecular-cross-validation@04d9df0 git+https://github.com/jorvis/Multicore-TSNE@6832575 git+https://github.com/KrishnaswamyLab/harmonic-alignment@v0.1#subdirectory=python +git+https://github.com/michalk8/neuralee@8946abf # contains gradient error fix git+https://github.com/scottgigante-immunai/knn-smoothing@python_package magic-impute==3.0.* phate==1.0.* +pybedtools==0.9.* +pyensembl==2.0.* +scalex==1.0.2 +scvi-tools==0.16.* +tangram-sc==1.0.* +tensorflow-cpu==2.9.* +torch==1.12.* xgboost==1.6.* diff --git a/docker/openproblems-python-pytorch/requirements.txt b/docker/openproblems-python-pytorch/requirements.txt deleted file mode 100644 index 56bd2a53dc..0000000000 --- a/docker/openproblems-python-pytorch/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -git+https://github.com/BayraktarLab/cell2location.git@7e7aa231cc61ff460da14402fa3b9a1fa3ec69ac -git+https://github.com/czbiohub/molecular-cross-validation@04d9df0 -git+https://github.com/michalk8/neuralee@8946abf # contains gradient error fix -jax==0.3.23 -jaxlib==0.3.22 -scalex==1.0.2 -scikit-misc==0.1.* -scvi-tools~=0.17 # pinned in #313 -tangram-sc==1.0.* -torch==1.12.* -xgboost==1.6.* diff --git a/docker/openproblems-python-pytorch/Dockerfile b/docker/openproblems-python-scvi/Dockerfile similarity index 65% rename from docker/openproblems-python-pytorch/Dockerfile rename to docker/openproblems-python-scvi/Dockerfile index 0f5b8521a3..f7edd2e4dc 100644 --- a/docker/openproblems-python-pytorch/Dockerfile +++ b/docker/openproblems-python-scvi/Dockerfile @@ -7,8 +7,8 @@ ARG NB_GID="100" USER root WORKDIR / -# install dependencies and openproblems -COPY ./docker/openproblems-python-pytorch/requirements.txt ./requirements.txt +# Install Python packages +COPY ./docker/openproblems-python-scvi/requirements.txt ./requirements.txt RUN pip install --no-cache-dir -r requirements.txt USER $NB_UID diff --git a/docker/openproblems-python-pytorch/README.md b/docker/openproblems-python-scvi/README.md similarity index 64% rename from docker/openproblems-python-pytorch/README.md rename to docker/openproblems-python-scvi/README.md index d566a8efd5..546cec9bc0 100644 --- a/docker/openproblems-python-pytorch/README.md +++ b/docker/openproblems-python-scvi/README.md @@ -9,9 +9,3 @@ Python: 3.8 Python packages: * scvi-tools -* tangram -* torch -* neuralee -* xgboost -* molecular-cross-validation -* cell2location diff --git a/docker/openproblems-python-scvi/requirements.txt b/docker/openproblems-python-scvi/requirements.txt new file mode 100644 index 0000000000..87aa041993 --- /dev/null +++ b/docker/openproblems-python-scvi/requirements.txt @@ -0,0 +1,5 @@ +jax==0.3.23 +jaxlib==0.3.22 +scikit-misc==0.1.* +scvi-tools~=0.17 # pinned in #313 +xgboost==1.6.* diff --git a/docker/openproblems-python-tensorflow/Dockerfile b/docker/openproblems-python-tensorflow/Dockerfile deleted file mode 100644 index 0a996f9221..0000000000 --- a/docker/openproblems-python-tensorflow/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM singlecellopenproblems/openproblems:latest - -ARG NB_USER="sagemaker-user" -ARG NB_UID="1000" -ARG NB_GID="100" - -USER root -WORKDIR / - -# install dependencies and openproblems -COPY ./docker/openproblems-python-tensorflow/requirements.txt ./requirements.txt -RUN pip install --no-cache-dir -r requirements.txt - -USER $NB_UID -WORKDIR /home/$NB_USER diff --git a/docker/openproblems-python-tensorflow/requirements.txt b/docker/openproblems-python-tensorflow/requirements.txt deleted file mode 100644 index f2a476acf9..0000000000 --- a/docker/openproblems-python-tensorflow/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -git+https://github.com/Avsecz/kopt@6a5c890 # master -git+https://github.com/scottgigante-immunai/dca@1f4edbc # patch-1 contains tf version bump -protobuf==3.20.* -tensorflow==2.9.0 diff --git a/docker/openproblems-python-tf2.4/Dockerfile b/docker/openproblems-python-tf2.4/Dockerfile new file mode 100644 index 0000000000..dcdabaf28d --- /dev/null +++ b/docker/openproblems-python-tf2.4/Dockerfile @@ -0,0 +1,18 @@ +FROM singlecellopenproblems/openproblems:latest + +ARG NB_USER="sagemaker-user" +ARG NB_UID="1000" +ARG NB_GID="100" + +USER root +WORKDIR / + +# Install Python packages +COPY ./docker/openproblems-python-tf2.4/requirements.txt ./requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +# tensorflow downgrades numpy and h5py (and therefore anndata) +RUN pip install --no-cache-dir -e /usr/src/singlecellopenproblems + +USER $NB_UID +WORKDIR /home/$NB_USER diff --git a/docker/openproblems-python-tensorflow/README.md b/docker/openproblems-python-tf2.4/README.md similarity index 56% rename from docker/openproblems-python-tensorflow/README.md rename to docker/openproblems-python-tf2.4/README.md index 3f4c4dbf67..f08a69ea20 100644 --- a/docker/openproblems-python-tensorflow/README.md +++ b/docker/openproblems-python-tf2.4/README.md @@ -1,4 +1,4 @@ -# openproblems-python-tensorflow Docker image +# openproblems-python-tf2.4 Docker image Base image: singlecellopenproblems/openproblems @@ -8,5 +8,6 @@ Python: 3.8 Python packages: -* tensorflow +* keras >=2.4,<2.6 +* tensorflow >=2.4,<2.5 * dca diff --git a/docker/openproblems-python-tf2.4/requirements.txt b/docker/openproblems-python-tf2.4/requirements.txt new file mode 100644 index 0000000000..31a56c2ea2 --- /dev/null +++ b/docker/openproblems-python-tf2.4/requirements.txt @@ -0,0 +1,4 @@ +dca==0.3.* +keras>=2.4,<2.11 +pyyaml==6.0 # pinned in #431 +tensorflow-cpu==2.4.* # pinned in dca diff --git a/docker/openproblems-r-base/README.md b/docker/openproblems-r-base/README.md index ebca77780d..785a9ace1b 100644 --- a/docker/openproblems-r-base/README.md +++ b/docker/openproblems-r-base/README.md @@ -28,4 +28,4 @@ R packages: Python packages: * rpy2 -* anndata2ri>=1.1 +* anndata2ri>=1.0.6 diff --git a/docker/openproblems-r-extras/r_requirements.txt b/docker/openproblems-r-extras/r_requirements.txt index 0b8da4bde9..efd4ceeba6 100644 --- a/docker/openproblems-r-extras/r_requirements.txt +++ b/docker/openproblems-r-extras/r_requirements.txt @@ -46,7 +46,6 @@ shiny@1.4.0.2 sparsesvd@0.2 systemfonts@1.0.4 textshaping@0.3.6 -theislab/kBET@a10ffea # master tibble@3.1.7 tidymodels@0.1.2 tidyverse@1.3.0 diff --git a/docker/openproblems-r-extras/requirements.txt b/docker/openproblems-r-extras/requirements.txt index 1795471bae..9d03431a90 100644 --- a/docker/openproblems-r-extras/requirements.txt +++ b/docker/openproblems-r-extras/requirements.txt @@ -1,3 +1,3 @@ git+https://github.com/KrishnaswamyLab/harmonic-alignment@v0.1#subdirectory=python -git+https://github.com/theislab/scib@f0be826 +git+https://github.com/theislab/scib@v1.0.2 xgboost==1.6.* diff --git a/docker/openproblems-r-pytorch/README.md b/docker/openproblems-r-pytorch/README.md index 33d3e5a30c..03ccbc07d4 100644 --- a/docker/openproblems-r-pytorch/README.md +++ b/docker/openproblems-r-pytorch/README.md @@ -8,12 +8,12 @@ Python: 3.8 R: 4.0 +R packages: + +* batchelor +* sparsesvd +* dplyr + Python packages: -* harmony-pytorch -* torch -* bbknn -* mnnpy -* scib -* scanorama -* scvi-tools +* harmonic-alignment diff --git a/docker/openproblems-r-pytorch/requirements.txt b/docker/openproblems-r-pytorch/requirements.txt index c9dfe890a9..e7c8df42b7 100644 --- a/docker/openproblems-r-pytorch/requirements.txt +++ b/docker/openproblems-r-pytorch/requirements.txt @@ -1,8 +1,3 @@ -annoy==1.17.1 -bbknn==1.5.* -git+https://github.com/scottgigante-immunai/mnnpy@eb4c551 # branch: patch-2 -git+https://github.com/theislab/scib@f0be826 +git+https://github.com/theislab/scib@v1.0.2 harmony-pytorch==0.1.* -scanorama==1.7.0 -scvi-tools~=0.16 # pinned in #313 torch==1.13.* diff --git a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/cc_score.py b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/cc_score.py index e83d47bb54..322891b202 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/cc_score.py +++ b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/cc_score.py @@ -21,7 +21,7 @@ @metric( metric_name="Cell Cycle Score", maximize=True, - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", # only if required ) def cc_score(adata, test=False): from ._utils import _get_split diff --git a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/iso_label_sil.py b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/iso_label_sil.py index c1f8c4be2d..c3575de5b8 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/iso_label_sil.py +++ b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/iso_label_sil.py @@ -15,7 +15,7 @@ @metric( metric_name="Isolated label Silhouette", maximize=True, - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", # only if required ) def isolated_labels_sil(adata): from scib.metrics import isolated_labels diff --git a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/pcr.py b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/pcr.py index 7efca62ffe..886f26078b 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/pcr.py +++ b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/pcr.py @@ -18,7 +18,7 @@ @metric( metric_name="PC Regression", maximize=True, - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", # only if required ) def pcr(adata): from ._utils import _get_split diff --git a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/sil_batch.py b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/sil_batch.py index 9f28cd1284..c02e5e42aa 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/sil_batch.py +++ b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/sil_batch.py @@ -24,7 +24,7 @@ @metric( metric_name="Batch ASW", maximize=True, - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", # only if required ) def silhouette_batch(adata): from scib.metrics import silhouette_batch diff --git a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/silhouette.py b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/silhouette.py index bb2bece193..36991e1d67 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/silhouette.py +++ b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/silhouette.py @@ -12,7 +12,7 @@ @metric( metric_name="Silhouette", maximize=True, - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", # only if required ) def silhouette(adata): from scib.metrics import silhouette diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/hvg_conservation.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/hvg_conservation.py index bb7f90cae8..d40b36b740 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/hvg_conservation.py +++ b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/hvg_conservation.py @@ -21,7 +21,7 @@ @metric( metric_name="HVG conservation", maximize=True, - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", ) def hvg_conservation(adata): from scib.metrics import hvg_overlap diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/bbknn.py b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/bbknn.py index 97570dccd8..017ca8f766 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/bbknn.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/bbknn.py @@ -10,7 +10,7 @@ paper_url="https://academic.oup.com/bioinformatics/article/36/3/964/5545955", paper_year=2020, code_url="https://github.com/Teichlab/bbknn", - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", # only if required ) diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/combat.py b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/combat.py index 96e53538d3..3043a552e1 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/combat.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/combat.py @@ -10,7 +10,7 @@ paper_url="https://academic.oup.com/biostatistics/article/8/1/118/252073", paper_year=2007, code_url="https://scanpy.readthedocs.io/en/stable/api/scanpy.pp.combat.html", - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", ) diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/mnn.py b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/mnn.py index 0146f5b6e3..99dab39203 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/mnn.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/mnn.py @@ -10,7 +10,7 @@ paper_url="https://www.nature.com/articles/nbt.4091", paper_year=2018, code_url="https://github.com/chriscainx/mnnpy", - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", ) diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scalex.py b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scalex.py index 461ea04a94..36843b81ed 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scalex.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scalex.py @@ -11,7 +11,7 @@ paper_url="https://doi.org/10.1038/s41467-022-33758-z", paper_year=2022, code_url="https://github.com/jsxlei/SCALEX", - image="openproblems-python-pytorch", + image="openproblems-python-extras", ) diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanorama.py b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanorama.py index a5efc04b35..d6e80162b5 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanorama.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanorama.py @@ -10,7 +10,7 @@ paper_url="https://www.nature.com/articles/s41587-019-0113-3", paper_year=2019, code_url="https://github.com/brianhie/scanorama", - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", ) diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanvi.py b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanvi.py index 8f98a3c931..d5bf463974 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanvi.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanvi.py @@ -11,7 +11,7 @@ paper_url="https://doi.org/10.15252/msb.20209620", paper_year=2021, code_url="https://github.com/YosefLab/scvi-tools", - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", ) diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scvi.py b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scvi.py index 35f1cd7ac5..9e9a82a9f2 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scvi.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/methods/scvi.py @@ -10,7 +10,7 @@ paper_url="https://www.nature.com/articles/s41592-018-0229-2", paper_year=2018, code_url="https://github.com/YosefLab/scvi-tools", - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", # only if required ) diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/ari.py b/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/ari.py index 0d082fff44..13e7eb8ce1 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/ari.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/ari.py @@ -16,7 +16,7 @@ @metric( metric_name="ARI", maximize=True, - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", ) def ari(adata): from scib.metrics import ari diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/graph_connectivity.py b/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/graph_connectivity.py index 52dd7c44b2..3a9732d0e2 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/graph_connectivity.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/graph_connectivity.py @@ -22,7 +22,7 @@ @metric( metric_name="Graph connectivity", maximize=True, - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", # only if required ) def graph_connectivity(adata): import scib.metrics diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/iso_label_f1.py b/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/iso_label_f1.py index 71cd7ca209..df86b043d9 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/iso_label_f1.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/iso_label_f1.py @@ -27,7 +27,7 @@ @metric( metric_name="Isolated label F1", maximize=True, - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", # only if required ) def isolated_labels_f1(adata): from scib.metrics import isolated_labels diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/nmi.py b/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/nmi.py index 3356507b2e..bbcdc7cd9d 100644 --- a/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/nmi.py +++ b/openproblems/tasks/_batch_integration/batch_integration_graph/metrics/nmi.py @@ -15,7 +15,7 @@ @metric( metric_name="NMI", maximize=True, - image="openproblems-r-pytorch", + image="openproblems-python-batch-integration", ) def nmi(adata): from scib.metrics.clustering import opt_louvain # isort:skip diff --git a/openproblems/tasks/_cell_cell_communication/_common/methods/liana.py b/openproblems/tasks/_cell_cell_communication/_common/methods/liana.py index a528c9c9e3..dc2bba944c 100644 --- a/openproblems/tasks/_cell_cell_communication/_common/methods/liana.py +++ b/openproblems/tasks/_cell_cell_communication/_common/methods/liana.py @@ -43,10 +43,6 @@ def _liana( adata.layers["logcounts"] = adata.layers["log_cpm"] del adata.layers["log_cpm"] - # remove dataframe before R conversion - target = adata.uns["ccc_target"] - del adata.uns["ccc_target"] - # Run LIANA liana_res = _r_liana( adata, @@ -57,9 +53,6 @@ def _liana( **kwargs, ) - # return target to uns - adata.uns["ccc_target"] = target - # Format results liana_res["score"] = liana_res[score_col] adata.uns["ccc_pred"] = liana_res diff --git a/openproblems/tasks/denoising/datasets/pancreas.py b/openproblems/tasks/denoising/datasets/pancreas.py index bd039ad88e..c18ddbeee1 100644 --- a/openproblems/tasks/denoising/datasets/pancreas.py +++ b/openproblems/tasks/denoising/datasets/pancreas.py @@ -11,7 +11,7 @@ "across technologies (CEL-seq, CEL-seq2, Smart-seq2, inDrop, Fluidigm C1, " "and SMARTER-seq). Here we just use the inDrop1 batch, which includes" "1937 cells × 15502 genes.", - image="openproblems-python-pytorch", + image="openproblems-python-extras", ) def pancreas(test=False): adata = load_pancreas(test=test, keep_techs=["inDrop1"]) diff --git a/openproblems/tasks/denoising/datasets/pbmc.py b/openproblems/tasks/denoising/datasets/pbmc.py index 440ebe8a3b..fcf0fc782b 100644 --- a/openproblems/tasks/denoising/datasets/pbmc.py +++ b/openproblems/tasks/denoising/datasets/pbmc.py @@ -11,7 +11,7 @@ "1k Peripheral Blood Mononuclear Cells (PBMCs) from a healthy donor. " "Sequenced on 10X v3 chemistry in November 2018 by 10X Genomics." ), - image="openproblems-python-pytorch", + image="openproblems-python-extras", ) def pbmc(test=False): adata = load_tenx_1k_pbmc(test=test) diff --git a/openproblems/tasks/denoising/datasets/tabula_muris_senis.py b/openproblems/tasks/denoising/datasets/tabula_muris_senis.py index 9524cc4e95..a5c62f953e 100644 --- a/openproblems/tasks/denoising/datasets/tabula_muris_senis.py +++ b/openproblems/tasks/denoising/datasets/tabula_muris_senis.py @@ -10,7 +10,7 @@ dataset_summary="All lung cells from Tabula Muris Senis, a 500k cell-atlas from 18 " "organs and tissues across the mouse lifespan. Here we use just 10x data from lung." " 24540 cells × 16160 genes across 3 time points.", - image="openproblems-python-pytorch", + image="openproblems-python-extras", ) def tabula_muris_senis_lung_random(test=False): adata = load_tabula_muris_senis( diff --git a/openproblems/tasks/denoising/methods/dca.py b/openproblems/tasks/denoising/methods/dca.py index 458f9ecb05..fddfb85d51 100644 --- a/openproblems/tasks/denoising/methods/dca.py +++ b/openproblems/tasks/denoising/methods/dca.py @@ -30,7 +30,7 @@ def _dca(adata, test=False, epochs=None): paper_url="https://www.nature.com/articles/s41467-018-07931-2", paper_year=2019, code_url="https://github.com/theislab/dca", - image="openproblems-python-tensorflow", + image="openproblems-python-tf2.4", ) def dca(adata, test=False, epochs=None): return _dca(adata, test=test, epochs=epochs) diff --git a/openproblems/tasks/denoising/metrics/poisson.py b/openproblems/tasks/denoising/metrics/poisson.py index 93db71eee2..ebd2a73378 100644 --- a/openproblems/tasks/denoising/metrics/poisson.py +++ b/openproblems/tasks/denoising/metrics/poisson.py @@ -1,7 +1,7 @@ from ....tools.decorators import metric -@metric(metric_name="Poisson loss", maximize=False, image="openproblems-python-pytorch") +@metric(metric_name="Poisson loss", maximize=False, image="openproblems-python-extras") def poisson(adata): from molecular_cross_validation.mcv_sweep import poisson_nll_loss diff --git a/openproblems/tasks/dimensionality_reduction/methods/neuralee.py b/openproblems/tasks/dimensionality_reduction/methods/neuralee.py index 420132dbf2..41f55dbb65 100644 --- a/openproblems/tasks/dimensionality_reduction/methods/neuralee.py +++ b/openproblems/tasks/dimensionality_reduction/methods/neuralee.py @@ -18,7 +18,7 @@ paper_url="https://www.frontiersin.org/articles/10.3389/fgene.2020.00786/full", paper_year=2020, code_url="https://github.com/HiBearME/NeuralEE", - image="openproblems-python-pytorch", + image="openproblems-python-extras", ) diff --git a/openproblems/tasks/label_projection/methods/scvi_tools.py b/openproblems/tasks/label_projection/methods/scvi_tools.py index 1cec698b27..6212f8cd69 100644 --- a/openproblems/tasks/label_projection/methods/scvi_tools.py +++ b/openproblems/tasks/label_projection/methods/scvi_tools.py @@ -10,7 +10,7 @@ paper_url="https://doi.org/10.15252/msb.20209620", paper_year=2021, code_url="https://github.com/YosefLab/scvi-tools", - image="openproblems-python-pytorch", + image="openproblems-python-scvi", ) _scanvi_scarches_method = functools.partial( @@ -19,7 +19,7 @@ paper_url="https://doi.org/10.1101/2020.07.16.205997", paper_year=2021, code_url="https://github.com/YosefLab/scvi-tools", - image="openproblems-python-pytorch", + image="openproblems-python-scvi", ) diff --git a/openproblems/tasks/regulatory_effect_prediction/methods/beta.py b/openproblems/tasks/regulatory_effect_prediction/methods/beta.py index 1eeff4992e..e5dffb6fa7 100644 --- a/openproblems/tasks/regulatory_effect_prediction/methods/beta.py +++ b/openproblems/tasks/regulatory_effect_prediction/methods/beta.py @@ -235,7 +235,7 @@ def _beta(adata, test=False, top_genes=None, threshold=1): paper_year=2013, code_version="1.0", code_url="http://cistrome.org/BETA", - image="openproblems-python-bedtools", + image="openproblems-python-extras", ) def beta(adata, test=False, top_genes=None, threshold=1): adata = _beta(adata, test=test, top_genes=top_genes, threshold=threshold) diff --git a/openproblems/tasks/spatial_decomposition/datasets/destvi/generate.py b/openproblems/tasks/spatial_decomposition/datasets/destvi/generate.py index e446b07687..86005d07f2 100644 --- a/openproblems/tasks/spatial_decomposition/datasets/destvi/generate.py +++ b/openproblems/tasks/spatial_decomposition/datasets/destvi/generate.py @@ -11,7 +11,7 @@ "from the destVI manuscripts leveraging sparsePCA. Number of cells and " "cell types present in each spatial spot is computed via combination of " "kernel-based parametrization of a categorical distribution and the NB model.", - image="openproblems-python-pytorch", + image="openproblems-python-extras", ) def destvi(test=False): from .utils import generate_synthetic_dataset diff --git a/openproblems/tasks/spatial_decomposition/methods/cell2location.py b/openproblems/tasks/spatial_decomposition/methods/cell2location.py index 168472cbbb..2caf9d4956 100644 --- a/openproblems/tasks/spatial_decomposition/methods/cell2location.py +++ b/openproblems/tasks/spatial_decomposition/methods/cell2location.py @@ -12,7 +12,7 @@ paper_url="https://doi.org/10.1038/s41587-021-01139-4", paper_year=2022, code_url="https://github.com/BayraktarLab/cell2location", - image="openproblems-python-pytorch", + image="openproblems-python-extras", ) diff --git a/openproblems/tasks/spatial_decomposition/methods/destvi.py b/openproblems/tasks/spatial_decomposition/methods/destvi.py index 4338a465fd..9330ba1d92 100644 --- a/openproblems/tasks/spatial_decomposition/methods/destvi.py +++ b/openproblems/tasks/spatial_decomposition/methods/destvi.py @@ -11,7 +11,7 @@ paper_url="https://doi.org/10.1038/s41587-022-01272-8", paper_year=2022, code_url="https://github.com/YosefLab/scvi-tools", - image="openproblems-python-pytorch", + image="openproblems-python-extras", ) def destvi( adata, diff --git a/openproblems/tasks/spatial_decomposition/methods/stereoscope.py b/openproblems/tasks/spatial_decomposition/methods/stereoscope.py index f9c025319e..c2695e4253 100644 --- a/openproblems/tasks/spatial_decomposition/methods/stereoscope.py +++ b/openproblems/tasks/spatial_decomposition/methods/stereoscope.py @@ -10,7 +10,7 @@ paper_url="https://doi.org/10.1038/s41587-022-01272-8", paper_year=2020, code_url="https://github.com/scverse/scvi-tools", - image="openproblems-python-pytorch", + image="openproblems-python-extras", ) def stereoscope(adata, test=False, max_epochs_sc=None, max_epochs_sp=None): from scvi.external import RNAStereoscope diff --git a/openproblems/tasks/spatial_decomposition/methods/tangram.py b/openproblems/tasks/spatial_decomposition/methods/tangram.py index c05eb0339f..329676c4aa 100644 --- a/openproblems/tasks/spatial_decomposition/methods/tangram.py +++ b/openproblems/tasks/spatial_decomposition/methods/tangram.py @@ -10,7 +10,7 @@ paper_url="https://doi.org/10.1038/s41592-021-01264-7", paper_year=2021, code_url="https://github.com/broadinstitute/Tangram", - image="openproblems-python-pytorch", + image="openproblems-python-extras", ) def tangram(adata, test=False, num_epochs=None, n_markers=None): # analysis based on: diff --git a/pytest.ini b/pytest.ini index 273a414dbc..447d92b2a1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,5 +10,4 @@ filterwarnings = ignore:X\.dtype being converted to np\.float32 from float64:FutureWarning ignore:is_categorical is deprecated and will be removed in a future version:FutureWarning ignore:The use of (converter|py2rpy|rpy2py) in module rpy2.robjects.conversion is deprecated.:DeprecationWarning - ignore:`Model\.state_updates` will be removed in a future version\.:UserWarning always:Container failed with AssertionError\. Retrying [0-9]* more time:RuntimeWarning diff --git a/setup.py b/setup.py index 89ead383cd..a6a1c213f4 100644 --- a/setup.py +++ b/setup.py @@ -4,14 +4,14 @@ import os install_requires = [ - "numpy>=1.19.2,<1.24", - "scikit-learn<=1.1.*", + "numpy>=1.22,<1.24", + "scikit-learn==1.1.*", "anndata==0.8.*", "scprep>=1.2.1", - "scipy>=1.7,<1.10", + "scipy>=1.8,<1.10", "scanpy>=1.6", - "louvain==0.8.*", - "python-igraph==0.10.*", + "louvain==0.7.*", + "python-igraph<0.10", "decorator<5.0", # pinned in #324 "memory-profiler==0.60", "colorama==0.4.*", diff --git a/workflow/Snakefile b/workflow/Snakefile index 6f02c7fe2b..c8378a3cea 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -27,7 +27,6 @@ rule docker_refresh: input: tools.refresh_images rule refresh_docker_image: - threads: 0 priority: 50 input: dockerfile = "{}/{{image}}/refresh.Dockerfile".format(tools.IMAGES_DIR), @@ -79,7 +78,6 @@ rule update_docker_image: "touch {output}" rule build_docker_image: - threads: 0.5 input: dockerfile = "{}/{{image}}/Dockerfile".format(tools.IMAGES_DIR), requirements = tools.docker_build_requirements, @@ -110,7 +108,6 @@ rule login_docker: "docker login --username=singlecellopenproblems --password=$(cat {input})" rule push_docker_image: - threads: 0 input: build = "{}/{{image}}/.docker_update".format(tools.IMAGES_DIR), login = ".docker_login", @@ -120,7 +117,6 @@ rule push_docker_image: "docker push --quiet singlecellopenproblems/{wildcards.image}" rule pull_docker_image: - threads: 0 output: temp(touch("{}/{{image}}/.docker_pull".format(tools.IMAGES_DIR))) shell: diff --git a/workflow/snakemake_tools.py b/workflow/snakemake_tools.py index 02d327dc55..8c9b92e42b 100644 --- a/workflow/snakemake_tools.py +++ b/workflow/snakemake_tools.py @@ -264,7 +264,6 @@ def docker_image_label(image, label): return output -@functools.lru_cache(None) def docker_imagespec_changed(image, dockerfile): """Check if the Dockerfile has changed @@ -274,17 +273,6 @@ def docker_imagespec_changed(image, dockerfile): If working with a github actions-built image, check if there is any diff between the Dockerfile and base/main """ - base_image = _docker_base(image) - if base_image is not None: - base_docker_path = os.path.join(IMAGES_DIR, base_image) - base_dockerfile = os.path.join(base_docker_path, "Dockerfile") - if docker_imagespec_changed(base_image, base_dockerfile): - print( - "{}: base image spec changed".format(image), - file=sys.stderr, - ) - return True - if not docker_image_exists(image): # will be downloaded from dockerhub build_type = "github_actions"