Run pip check on build (#678)

* squash

* revert changes to installation

* building takes half a cpu

.github/workflows/run_tests.yml

@@ -62,13 +62,6 @@ jobs:
         pip install --editable .[evaluate]
         python -c "import openproblems"
 
-    - name: Download docker images
-      run: |
-        for image in $(find ./docker -mindepth 1 -type d -exec basename {} \;); do
-          docker pull singlecellopenproblems/${image} &
-        done
-        wait
-
     - name: Update Docker images
       if: |
         !(

docker/openproblems-python-bedtools/Dockerfile

@@ -0,0 +1,28 @@
+FROM singlecellopenproblems/openproblems:latest
+
+ARG NB_USER="sagemaker-user"
+ARG NB_UID="1000"
+ARG NB_GID="100"
+
+USER root
+WORKDIR /
+
+# Install pybedtools dependency
+ARG BUILD_PACKAGES=""
+ARG PACKAGE_VERSION=2.27.1
+RUN apt-get update && \
+    apt-get install --yes git openssl build-essential zlib1g-dev && \
+    cd /tmp && \
+    git clone https://github.com/arq5x/bedtools2.git && \
+    cd bedtools2 && \
+    git checkout v$PACKAGE_VERSION && \
+    make && \
+    mv bin/* /usr/local/bin && \
+    cd /
+
+# install dependencies and openproblems
+COPY ./docker/openproblems-python-bedtools/requirements.txt ./requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+USER $NB_UID
+WORKDIR /home/$NB_USER

docker/openproblems-python-bedtools/README.md

@@ -0,0 +1,12 @@
+# openproblems-python-extras Docker image
+
+Base image: singlecellopenproblems/openproblems
+
+OS: Debian Stretch
+
+Python: 3.8
+
+Python packages:
+
+* pybedtools
+* pyensembl

docker/openproblems-python-bedtools/requirements.txt

@@ -0,0 +1,2 @@
+pybedtools==0.9.*
+pyensembl==2.0.*

docker/openproblems-python-extras/requirements.txt

@@ -1,17 +1,7 @@
 cmake==3.24.1.1
-git+https://github.com/BayraktarLab/cell2location.git@7e7aa231cc61ff460da14402fa3b9a1fa3ec69ac
-git+https://github.com/czbiohub/molecular-cross-validation@04d9df0
 git+https://github.com/jorvis/Multicore-TSNE@6832575
 git+https://github.com/KrishnaswamyLab/[email protected]#subdirectory=python
-git+https://github.com/michalk8/neuralee@8946abf  # contains gradient error fix
 git+https://github.com/scottgigante-immunai/knn-smoothing@python_package
 magic-impute==3.0.*
 phate==1.0.*
-pybedtools==0.9.*
-pyensembl==2.0.*
-scalex==1.0.2
-scvi-tools==0.16.*
-tangram-sc==1.0.*
-tensorflow-cpu==2.9.*
-torch==1.12.*
 xgboost==1.6.*

docker/openproblems-python-scvi/Dockerfile → docker/openproblems-python-pytorch/Dockerfile

@@ -7,8 +7,8 @@ ARG NB_GID="100"
 USER root
 WORKDIR /
 
-# Install Python packages
-COPY ./docker/openproblems-python-scvi/requirements.txt ./requirements.txt
+# install dependencies and openproblems
+COPY ./docker/openproblems-python-pytorch/requirements.txt ./requirements.txt
 RUN pip install --no-cache-dir -r requirements.txt
 
 USER $NB_UID

docker/openproblems-python-scvi/README.md → docker/openproblems-python-pytorch/README.md

@@ -9,3 +9,9 @@ Python: 3.8
 Python packages:
 
 * scvi-tools
+* tangram
+* torch
+* neuralee
+* xgboost
+* molecular-cross-validation
+* cell2location

docker/openproblems-python-pytorch/requirements.txt

@@ -0,0 +1,11 @@
+git+https://github.com/BayraktarLab/cell2location.git@7e7aa231cc61ff460da14402fa3b9a1fa3ec69ac
+git+https://github.com/czbiohub/molecular-cross-validation@04d9df0
+git+https://github.com/michalk8/neuralee@8946abf  # contains gradient error fix
+jax==0.3.23
+jaxlib==0.3.22
+scalex==1.0.2
+scikit-misc==0.1.*
+scvi-tools~=0.17  # pinned in #313
+tangram-sc==1.0.*
+torch==1.12.*
+xgboost==1.6.*

docker/openproblems-python-tensorflow/Dockerfile

@@ -0,0 +1,15 @@
+FROM singlecellopenproblems/openproblems:latest
+
+ARG NB_USER="sagemaker-user"
+ARG NB_UID="1000"
+ARG NB_GID="100"
+
+USER root
+WORKDIR /
+
+# install dependencies and openproblems
+COPY ./docker/openproblems-python-tensorflow/requirements.txt ./requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+USER $NB_UID
+WORKDIR /home/$NB_USER

docker/openproblems-python-tf2.4/README.md → docker/openproblems-python-tensorflow/README.md

@@ -1,4 +1,4 @@
-# openproblems-python-tf2.4 Docker image
+# openproblems-python-tensorflow Docker image
 
 Base image: singlecellopenproblems/openproblems
 
@@ -8,6 +8,5 @@ Python: 3.8
 
 Python packages:
 
-* keras >=2.4,<2.6
-* tensorflow >=2.4,<2.5
+* tensorflow
 * dca

docker/openproblems-python-tensorflow/requirements.txt

@@ -0,0 +1,4 @@
+git+https://github.com/Avsecz/kopt@6a5c890  # master
+git+https://github.com/scottgigante-immunai/dca@1f4edbc  # patch-1 contains tf version bump
+protobuf==3.20.*
+tensorflow==2.9.0

docker/openproblems-r-base/README.md

@@ -28,4 +28,4 @@ R packages:
 Python packages:
 
 * rpy2
-* anndata2ri>=1.0.6
+* anndata2ri>=1.1

docker/openproblems-r-extras/r_requirements.txt

@@ -46,6 +46,7 @@ [email protected]
 [email protected]
 [email protected]
 [email protected]
+theislab/kBET@a10ffea # master
 [email protected]
 [email protected]
 [email protected]

docker/openproblems-r-extras/requirements.txt

@@ -1,3 +1,3 @@
 git+https://github.com/KrishnaswamyLab/[email protected]#subdirectory=python
-git+https://github.com/theislab/scib@v1.0.2
+git+https://github.com/theislab/scib@f0be826
 xgboost==1.6.*

docker/openproblems-r-pytorch/README.md

@@ -8,12 +8,12 @@ Python: 3.8
 
 R: 4.0
 
-R packages:
-
-* batchelor
-* sparsesvd
-* dplyr
-
 Python packages:
 
-* harmonic-alignment
+* harmony-pytorch
+* torch
+* bbknn
+* mnnpy
+* scib
+* scanorama
+* scvi-tools

docker/openproblems-r-pytorch/requirements.txt

@@ -1,3 +1,8 @@
-git+https://github.com/theislab/[email protected]
+annoy==1.17.1
+bbknn==1.5.*
+git+https://github.com/scottgigante-immunai/mnnpy@eb4c551  # branch: patch-2
+git+https://github.com/theislab/scib@f0be826
 harmony-pytorch==0.1.*
+scanorama==1.7.0
+scvi-tools~=0.16  # pinned in #313
 torch==1.13.*

openproblems/tasks/_batch_integration/batch_integration_embed/metrics/cc_score.py

@@ -21,7 +21,7 @@
 @metric(
     metric_name="Cell Cycle Score",
     maximize=True,
-    image="openproblems-python-batch-integration",  # only if required
+    image="openproblems-r-pytorch",
 )
 def cc_score(adata, test=False):
     from ._utils import _get_split

openproblems/tasks/_batch_integration/batch_integration_embed/metrics/iso_label_sil.py

@@ -15,7 +15,7 @@
 @metric(
     metric_name="Isolated label Silhouette",
     maximize=True,
-    image="openproblems-python-batch-integration",  # only if required
+    image="openproblems-r-pytorch",
 )
 def isolated_labels_sil(adata):
     from scib.metrics import isolated_labels

openproblems/tasks/_batch_integration/batch_integration_embed/metrics/pcr.py

@@ -18,7 +18,7 @@
 @metric(
     metric_name="PC Regression",
     maximize=True,
-    image="openproblems-python-batch-integration",  # only if required
+    image="openproblems-r-pytorch",
 )
 def pcr(adata):
     from ._utils import _get_split

openproblems/tasks/_batch_integration/batch_integration_embed/metrics/sil_batch.py

@@ -24,7 +24,7 @@
 @metric(
     metric_name="Batch ASW",
     maximize=True,
-    image="openproblems-python-batch-integration",  # only if required
+    image="openproblems-r-pytorch",
 )
 def silhouette_batch(adata):
     from scib.metrics import silhouette_batch

openproblems/tasks/_batch_integration/batch_integration_embed/metrics/silhouette.py

@@ -12,7 +12,7 @@
 @metric(
     metric_name="Silhouette",
     maximize=True,
-    image="openproblems-python-batch-integration",  # only if required
+    image="openproblems-r-pytorch",
 )
 def silhouette(adata):
     from scib.metrics import silhouette

openproblems/tasks/_batch_integration/batch_integration_feature/metrics/hvg_conservation.py

@@ -21,7 +21,7 @@
 @metric(
     metric_name="HVG conservation",
     maximize=True,
-    image="openproblems-python-batch-integration",
+    image="openproblems-r-pytorch",
 )
 def hvg_conservation(adata):
     from scib.metrics import hvg_overlap

openproblems/tasks/_batch_integration/batch_integration_graph/methods/bbknn.py

@@ -10,7 +10,7 @@
     paper_url="https://academic.oup.com/bioinformatics/article/36/3/964/5545955",
     paper_year=2020,
     code_url="https://github.com/Teichlab/bbknn",
-    image="openproblems-python-batch-integration",  # only if required
+    image="openproblems-r-pytorch",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/combat.py

@@ -10,7 +10,7 @@
     paper_url="https://academic.oup.com/biostatistics/article/8/1/118/252073",
     paper_year=2007,
     code_url="https://scanpy.readthedocs.io/en/stable/api/scanpy.pp.combat.html",
-    image="openproblems-python-batch-integration",
+    image="openproblems-r-pytorch",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/mnn.py

@@ -10,7 +10,7 @@
     paper_url="https://www.nature.com/articles/nbt.4091",
     paper_year=2018,
     code_url="https://github.com/chriscainx/mnnpy",
-    image="openproblems-python-batch-integration",
+    image="openproblems-r-pytorch",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/scalex.py

@@ -11,7 +11,7 @@
     paper_url="https://doi.org/10.1038/s41467-022-33758-z",
     paper_year=2022,
     code_url="https://github.com/jsxlei/SCALEX",
-    image="openproblems-python-extras",
+    image="openproblems-python-pytorch",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanorama.py

@@ -10,7 +10,7 @@
     paper_url="https://www.nature.com/articles/s41587-019-0113-3",
     paper_year=2019,
     code_url="https://github.com/brianhie/scanorama",
-    image="openproblems-python-batch-integration",
+    image="openproblems-r-pytorch",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/scanvi.py

@@ -11,7 +11,7 @@
     paper_url="https://doi.org/10.15252/msb.20209620",
     paper_year=2021,
     code_url="https://github.com/YosefLab/scvi-tools",
-    image="openproblems-python-batch-integration",
+    image="openproblems-r-pytorch",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/methods/scvi.py

@@ -10,7 +10,7 @@
     paper_url="https://www.nature.com/articles/s41592-018-0229-2",
     paper_year=2018,
     code_url="https://github.com/YosefLab/scvi-tools",
-    image="openproblems-python-batch-integration",  # only if required
+    image="openproblems-r-pytorch",
 )
 
 

openproblems/tasks/_batch_integration/batch_integration_graph/metrics/ari.py

@@ -16,7 +16,7 @@
 @metric(
     metric_name="ARI",
     maximize=True,
-    image="openproblems-python-batch-integration",
+    image="openproblems-r-pytorch",
 )
 def ari(adata):
     from scib.metrics import ari