Resolves #23 (#53)

* Resolves #23 Avoids vectorization to avoid tensor shape errors * Update setup.cfg * Update setup.cfg * Fix linting * Try again * Do not install types * Try again * Try again * bump ubuntu version * Bump Python version 3.6 -> 3.7 * Make workflow more similar to Pyro * Use https in git_clone.py * Add pytest-xdist to dependendencies * Fix pytorch usage bug * pin protobuf version * clone nextclade in test target * Fix protobuf version, skip data-intensive test --------- Co-authored-by: Ben Kotzen <[email protected]>
broadinstitute · Feb 2, 2023 · a2b5c05 · a2b5c05
1 parent 95ce2da
commit a2b5c05
Show file tree

Hide file tree

Showing 11 changed files with 30 additions and 21 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.6]
+        python-version: [3.7]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -37,7 +37,7 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        python-version: [3.6]
+        python-version: [3.7]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -50,7 +50,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install gcc-8 g++-8 ninja-build
           python -m pip install --upgrade pip wheel setuptools
-          pip install torch==1.9.0+cpu torchvision==0.10.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
+          pip install torch==1.11.0+cpu torchvision==0.12.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
           pip install .[test]
           pip install coveralls
           pip freeze

diff --git a/Makefile b/Makefile
@@ -20,7 +20,7 @@ lint: FORCE
 	black --extend-exclude='notebooks|pyrocov/external' --check .
 	isort --check --skip=pyrocov/external .
 	python scripts/update_headers.py --check
-	mypy .
+	mypy . --exclude=build/
 
 format: FORCE
 	black --extend-exclude='notebooks|pyrocov/external' .
@@ -30,8 +30,11 @@ format: FORCE
 test: lint FORCE
 	python scripts/git_pull.py --no-update cov-lineages/pango-designation
 	python scripts/git_pull.py --no-update cov-lineages/pangoLEARN
+	python scripts/git_pull.py --no-update nextstrain/nextclade
 	pytest -v -n auto test
-	test -e results/aligndb && python scripts/mutrans.py --test -n 2 -s 2
+	test -e results/columns.3000.pkl \
+	  && python scripts/mutrans.py --test -n 2 -s 2 \
+	  || echo skipping test
 
 ###########################################################################
 # Main processing workflows

diff --git a/generate_epiToPublicAndDate.py b/generate_epiToPublicAndDate.py
@@ -1,3 +1,6 @@
+# Copyright Contributors to the Pyro-Cov project.
+# SPDX-License-Identifier: Apache-2.0
+
 #!/usr/bin/env python
 # coding: utf-8
 
@@ -8,7 +11,6 @@
 
 import pandas as pd
 
-
 # In[2]:
 
 
@@ -18,7 +20,7 @@
 # In[3]:
 
 
-gisaid_meta["vname"] = gisaid_meta["Virus name"].str.replace("hCoV-19/","")
+gisaid_meta["vname"] = gisaid_meta["Virus name"].str.replace("hCoV-19/", "")
 gisaid_meta["vname2"] = gisaid_meta["vname"]
 
 
@@ -31,7 +33,7 @@
 # In[5]:
 
 
-epi_map = epi_map.sort_values(by="Accession ID", ascending = True)
+epi_map = epi_map.sort_values(by="Accession ID", ascending=True)
 
 
 # In[6]:
@@ -43,7 +45,9 @@
 # In[7]:
 
 
-epi_map.to_csv("results/gisaid/epiToPublicAndDate.latest", header=False, sep="\t", index=False)
+epi_map.to_csv(
+    "results/gisaid/epiToPublicAndDate.latest", header=False, sep="\t", index=False
+)
 
 
 # In[8]:
@@ -55,12 +59,7 @@
 # In[9]:
 
 
-get_ipython().run_line_magic('pinfo', 'pd.DataFrame.to_csv')
+get_ipython().run_line_magic("pinfo", "pd.DataFrame.to_csv")  # type: ignore
 
 
 # In[ ]:
-
-
-
-
-
diff --git a/pyrocov/external/usher/parsimony_pb2.py b/pyrocov/external/usher/parsimony_pb2.py
diff --git a/pyrocov/geo.py b/pyrocov/geo.py
@@ -319,8 +319,8 @@ def gisaid_to_jhu_location(
             value = GISAID_TO_JHU[value[0]] + value[1:]
         while value not in jhu_prefixes:
             value = value[:-1]
-#            if not value:
-#                raise ValueError(f"Failed to find GISAID loctaion '{key}' in JHU data")
+        #            if not value:
+        #                raise ValueError(f"Failed to find GISAID loctaion '{key}' in JHU data")
         gisaid_to_jhu[key] = value
 
     # Construct a matrix many-to-many matching GISAID locations to JHU locations.

diff --git a/pyrocov/mutrans.py b/pyrocov/mutrans.py
@@ -932,6 +932,7 @@ def optim_config(param_name):
         model_type,
         num_samples=num_samples,
         forecast_steps=forecast_steps,
+        vectorize=False,  # vectorization probably won't work here
     )
     result["ELL"] = ell
     result["losses"] = losses

diff --git a/pyrocov/ops.py b/pyrocov/ops.py
@@ -74,7 +74,7 @@ def log_factorial_sum(x: torch.Tensor) -> torch.Tensor:
         return (x + 1).lgamma().sum()
     key = id(x)
     if key not in _log_factorial_cache:
-        weakref.finalize(x, _log_factorial_cache.pop, key, None)
+        weakref.finalize(x, _log_factorial_cache.pop, key, None)  # type: ignore
         _log_factorial_cache[key] = (x + 1).lgamma().sum()
     return _log_factorial_cache[key]
 

diff --git a/pyrocov/phylo.py b/pyrocov/phylo.py
@@ -125,7 +125,7 @@ def time_mrca(self):
                 d2 = descendents[c2]
                 mask = d1 & d2[:, None]
                 mask[p] |= d1 | d2
-                mask |= mask.T
+                mask |= mask.T.clone()
                 result[mask] = self.times[p]
         return result
 

diff --git a/scripts/git_pull.py b/scripts/git_pull.py
@@ -30,7 +30,9 @@
     dirname = os.path.join(dirname, repo)
     if not os.path.exists(dirname):
         print(f"Cloning {arg}")
-        check_call(["git", "clone", "--depth", "1", f"[email protected]:{user}/{repo}"])
+        check_call(
+            ["git", "clone", "--depth", "1", f"https://github.com/{user}/{repo}"]
+        )
     elif update:
         print(f"Pulling {arg}")
         os.chdir(dirname)

diff --git a/setup.cfg b/setup.cfg
@@ -3,6 +3,8 @@ max-line-length = 120
 ignore = E741,E203,W503
 per_file_ignores =
     pyrocov/io.py:E226
+exclude =
+    generate_epiToPublicAndDate.py
 
 [isort]
 profile = black

diff --git a/setup.py b/setup.py
@@ -38,7 +38,7 @@
         "scikit-learn",
         "umap-learn",
         "mappy",
-        "protobuf",
+        "protobuf>=3.12,<3.13",  # pinned by usher
         "tqdm",
         "colorcet",
     ],
@@ -48,6 +48,7 @@
             "isort>=5.0",
             "flake8",
             "pytest>=5.0",
+            "pytest-xdist",
             "mypy>=0.812",
             "types-protobuf",
         ],