Revert "Run pip check on build (#678)"

This reverts commit c8c08c8.

.github/workflows/run_tests.yml

@@ -62,6 +62,13 @@ jobs:
         pip install --editable .[evaluate]
         python -c "import openproblems"
 
+    - name: Download docker images
+      run: |
+        for image in $(find ./docker -mindepth 1 -type d -exec basename {} \;); do
+          docker pull singlecellopenproblems/${image} &
+        done
+        wait
+
     - name: Update Docker images
       if: |
         !(

docker/openproblems-python-batch-integration/Dockerfile

@@ -0,0 +1,24 @@
+FROM singlecellopenproblems/openproblems-r-base:latest
+
+USER root
+WORKDIR /
+
+ARG NB_USER="sagemaker-user"
+ARG NB_UID="1000"
+ARG NB_GID="100"
+
+RUN sed -i '$ d' /etc/apt/sources.list
+RUN \
+apt-get update --allow-releaseinfo-change && \
+apt-get -y install --no-install-recommends gcc git python3-llvmlite && \
+apt-get autoremove -y && \
+rm -rf /var/lib/apt/lists/*
+
+# Install Python packages
+COPY ./docker/openproblems-python-batch-integration/requirements.txt ./requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+# force reinstall annoy addresses https://github.com/spotify/annoy/issues/513
+RUN pip install --no-cache-dir --force annoy==1.17.0
+
+USER $NB_UID
+WORKDIR /home/$NB_USER

docker/openproblems-python-batch-integration/README.md

@@ -0,0 +1,15 @@
+# openproblems-python-extras Docker image
+
+Base image: singlecellopenproblems/openproblems-r-base
+
+OS: Debian Stretch
+
+Python: 3.8
+
+Python packages:
+
+* scIB
+* mnnpy
+* scanorama
+* bbknn
+* scVI

docker/openproblems-python-batch-integration/requirements.txt

@@ -0,0 +1,6 @@
+annoy==1.17.1
+bbknn==1.5.*
+git+https://github.com/scottgigante-immunai/mnnpy@eb4c551  # branch: patch-2
+git+https://github.com/theislab/scib@77ab015
+scanorama==1.7.0
+scvi-tools~=0.16  # pinned in #313

docker/openproblems-python-extras/requirements.txt

@@ -1,7 +1,17 @@
 cmake==3.24.1.1
+git+https://github.com/BayraktarLab/cell2location.git@7e7aa231cc61ff460da14402fa3b9a1fa3ec69ac
+git+https://github.com/czbiohub/molecular-cross-validation@04d9df0
 git+https://github.com/jorvis/Multicore-TSNE@6832575
 git+https://github.com/KrishnaswamyLab/[email protected]#subdirectory=python
+git+https://github.com/michalk8/neuralee@8946abf  # contains gradient error fix
 git+https://github.com/scottgigante-immunai/knn-smoothing@python_package
 magic-impute==3.0.*
 phate==1.0.*
+pybedtools==0.9.*
+pyensembl==2.0.*
+scalex==1.0.2
+scvi-tools==0.16.*
+tangram-sc==1.0.*
+tensorflow-cpu==2.9.*
+torch==1.12.*
 xgboost==1.6.*

docker/openproblems-python-pytorch/Dockerfile → docker/openproblems-python-scvi/Dockerfile

@@ -7,8 +7,8 @@ ARG NB_GID="100"
 USER root
 WORKDIR /
 
-# install dependencies and openproblems
-COPY ./docker/openproblems-python-pytorch/requirements.txt ./requirements.txt
+# Install Python packages
+COPY ./docker/openproblems-python-scvi/requirements.txt ./requirements.txt
 RUN pip install --no-cache-dir -r requirements.txt
 
 USER $NB_UID

docker/openproblems-python-pytorch/README.md → docker/openproblems-python-scvi/README.md

@@ -9,9 +9,3 @@ Python: 3.8
 Python packages:
 
 * scvi-tools
-* tangram
-* torch
-* neuralee
-* xgboost
-* molecular-cross-validation
-* cell2location

docker/openproblems-python-scvi/requirements.txt

@@ -0,0 +1,5 @@
+jax==0.3.23
+jaxlib==0.3.22
+scikit-misc==0.1.*
+scvi-tools~=0.17  # pinned in #313
+xgboost==1.6.*

docker/openproblems-python-tf2.4/Dockerfile

@@ -0,0 +1,18 @@
+FROM singlecellopenproblems/openproblems:latest
+
+ARG NB_USER="sagemaker-user"
+ARG NB_UID="1000"
+ARG NB_GID="100"
+
+USER root
+WORKDIR /
+
+# Install Python packages
+COPY ./docker/openproblems-python-tf2.4/requirements.txt ./requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+# tensorflow downgrades numpy and h5py (and therefore anndata)
+RUN pip install --no-cache-dir -e /usr/src/singlecellopenproblems
+
+USER $NB_UID
+WORKDIR /home/$NB_USER

docker/openproblems-python-tensorflow/README.md → docker/openproblems-python-tf2.4/README.md

@@ -1,4 +1,4 @@
-# openproblems-python-tensorflow Docker image
+# openproblems-python-tf2.4 Docker image
 
 Base image: singlecellopenproblems/openproblems
 
@@ -8,5 +8,6 @@ Python: 3.8
 
 Python packages:
 
-* tensorflow
+* keras >=2.4,<2.6
+* tensorflow >=2.4,<2.5
 * dca

docker/openproblems-python-tf2.4/requirements.txt

@@ -0,0 +1,4 @@
+dca==0.3.*
+keras>=2.4,<2.11
+pyyaml==6.0  # pinned in #431
+tensorflow-cpu==2.4.*  # pinned in dca

docker/openproblems-r-base/README.md

@@ -28,4 +28,4 @@ R packages:
 Python packages:
 
 * rpy2
-* anndata2ri>=1.1
+* anndata2ri>=1.0.6

docker/openproblems-r-extras/r_requirements.txt

@@ -46,7 +46,6 @@ [email protected]
 [email protected]
 [email protected]
 [email protected]
-theislab/kBET@a10ffea # master
 [email protected]
 [email protected]
 [email protected]

docker/openproblems-r-extras/requirements.txt

@@ -1,3 +1,3 @@
 git+https://github.com/KrishnaswamyLab/[email protected]#subdirectory=python
-git+https://github.com/theislab/scib@f0be826
+git+https://github.com/theislab/scib@v1.0.2
 xgboost==1.6.*

docker/openproblems-r-pytorch/README.md

@@ -8,12 +8,12 @@ Python: 3.8
 
 R: 4.0
 
+R packages:
+
+* batchelor
+* sparsesvd
+* dplyr
+
 Python packages:
 
-* harmony-pytorch
-* torch
-* bbknn
-* mnnpy
-* scib
-* scanorama
-* scvi-tools
+* harmonic-alignment

docker/openproblems-r-pytorch/requirements.txt

@@ -1,8 +1,3 @@
-annoy==1.17.1
-bbknn==1.5.*
-git+https://github.com/scottgigante-immunai/mnnpy@eb4c551  # branch: patch-2
-git+https://github.com/theislab/scib@f0be826
+git+https://github.com/theislab/[email protected]
 harmony-pytorch==0.1.*
-scanorama==1.7.0
-scvi-tools~=0.16  # pinned in #313
 torch==1.13.*

openproblems/tasks/_batch_integration/batch_integration_embed/metrics/cc_score.py

@@ -21,7 +21,7 @@
 @metric(
     metric_name="Cell Cycle Score",
     maximize=True,
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",  # only if required
 )
 def cc_score(adata, test=False):
     from ._utils import _get_split

openproblems/tasks/_batch_integration/batch_integration_embed/metrics/iso_label_sil.py

@@ -15,7 +15,7 @@
 @metric(
     metric_name="Isolated label Silhouette",
     maximize=True,
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",  # only if required
 )
 def isolated_labels_sil(adata):
     from scib.metrics import isolated_labels

openproblems/tasks/_batch_integration/batch_integration_embed/metrics/pcr.py

@@ -18,7 +18,7 @@
 @metric(
     metric_name="PC Regression",
     maximize=True,
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",  # only if required
 )
 def pcr(adata):
     from ._utils import _get_split

openproblems/tasks/_batch_integration/batch_integration_embed/metrics/sil_batch.py

@@ -24,7 +24,7 @@
 @metric(
     metric_name="Batch ASW",
     maximize=True,
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",  # only if required
 )
 def silhouette_batch(adata):
     from scib.metrics import silhouette_batch

openproblems/tasks/_batch_integration/batch_integration_embed/metrics/silhouette.py

@@ -12,7 +12,7 @@
 @metric(
     metric_name="Silhouette",
     maximize=True,
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",  # only if required
 )
 def silhouette(adata):
     from scib.metrics import silhouette

openproblems/tasks/_batch_integration/batch_integration_feature/metrics/hvg_conservation.py

@@ -21,7 +21,7 @@
 @metric(
     metric_name="HVG conservation",
     maximize=True,
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",
 )
 def hvg_conservation(adata):
     from scib.metrics import hvg_overlap

openproblems/tasks/_batch_integration/batch_integration_graph/methods/bbknn.py

@@ -10,7 +10,7 @@
     paper_url="https://academic.oup.com/bioinformatics/article/36/3/964/5545955",
     paper_year=2020,
     code_url="https://github.com/Teichlab/bbknn",
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",  # only if required
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/combat.py

@@ -10,7 +10,7 @@
     paper_url="https://academic.oup.com/biostatistics/article/8/1/118/252073",
     paper_year=2007,
     code_url="https://scanpy.readthedocs.io/en/stable/api/scanpy.pp.combat.html",
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/mnn.py

@@ -10,7 +10,7 @@
     paper_url="https://www.nature.com/articles/nbt.4091",
     paper_year=2018,
     code_url="https://github.com/chriscainx/mnnpy",
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/scalex.py

@@ -11,7 +11,7 @@
     paper_url="https://doi.org/10.1038/s41467-022-33758-z",
     paper_year=2022,
     code_url="https://github.com/jsxlei/SCALEX",
-    image="openproblems-python-pytorch",
+    image="openproblems-python-extras",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanorama.py

@@ -10,7 +10,7 @@
     paper_url="https://www.nature.com/articles/s41587-019-0113-3",
     paper_year=2019,
     code_url="https://github.com/brianhie/scanorama",
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanvi.py

@@ -11,7 +11,7 @@
     paper_url="https://doi.org/10.15252/msb.20209620",
     paper_year=2021,
     code_url="https://github.com/YosefLab/scvi-tools",
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/scvi.py

@@ -10,7 +10,7 @@
     paper_url="https://www.nature.com/articles/s41592-018-0229-2",
     paper_year=2018,
     code_url="https://github.com/YosefLab/scvi-tools",
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",  # only if required
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/metrics/ari.py

@@ -16,7 +16,7 @@
 @metric(
     metric_name="ARI",
     maximize=True,
-    image="openproblems-r-pytorch",
+    image="openproblems-python-batch-integration",
 )
 def ari(adata):
     from scib.metrics import ari