Merge pull request #176 from molgenis/feat/noGnomad

Remove Gnomad_HN and drop GRCh37 from the docs
molgenis · Feb 7, 2024 · e65e3d2 · e65e3d2
2 parents 63441b5 + 2f6c447
commit e65e3d2
Show file tree

Hide file tree

Showing 18 changed files with 22 additions and 38 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,8 +1,9 @@
 os: linux
-dist: focal
-language: python
+dist: jammy
+language: java
 python:
 - '3.10'
+jdk: openjdk17
 cache: pip
 branches:
   only:

diff --git a/README.md b/README.md
@@ -14,24 +14,12 @@ below.
 CAPICE can be used as online service at http://molgenis.org/capice
 
 ## Requirements
-The list below is a complete list. Depending on whether GRCh37 and/or GRCh38 is used and whether all
-mentioned features are used, some items in the list below can be skipped.
-
 * VEP v107
   * Including VEP cache (which needs to be unarchived!):
-    * [homo_sapiens_refseq_vep_107_GRCh37](http://ftp.ensembl.org/pub/release-107/variation/indexed_vep_cache/homo_sapiens_refseq_vep_107_GRCh37.tar.gz)
     * [homo_sapiens_refseq_vep_107_GRCh38](http://ftp.ensembl.org/pub/release-107/variation/indexed_vep_cache/homo_sapiens_refseq_vep_107_GRCh38.tar.gz)
   * Including plugin(s):
     * [Grantham](https://github.com/molgenis/vip/blob/master/resources/vep/plugins/Grantham.pm)
     * [SpliceAI](https://github.com/molgenis/vip/blob/master/resources/vep/plugins/SpliceAI.pm)
-  * Including additional data (GRCh37) [available here](https://download.molgeniscloud.org/downloads/vip/resources/GRCh37/):
-    * `gnomad.total.r2.1.1.sites.stripped.vcf.gz`
-    * `gnomad.total.r2.1.1.sites.stripped.vcf.gz.csi`
-    * `hg19.100way.phyloP100way.bw`
-    * `spliceai_scores.masked.indel.hg19.vcf.gz`
-    * `spliceai_scores.masked.indel.hg19.vcf.gz.tbi`
-    * `spliceai_scores.masked.snv.hg19.vcf.gz`
-    * `spliceai_scores.masked.snv.hg19.vcf.gz.tbi` 
   * Including additional data (GRCh38) [available here](https://download.molgeniscloud.org/downloads/vip/resources/GRCh38/):
     * `gnomad.genomes.v3.1.2.sites.stripped.vcf.gz`
     * `gnomad.genomes.v3.1.2.sites.stripped.vcf.gz.csi`
@@ -95,15 +83,14 @@ command:
 vep --input_file <path to your input file> --format vcf --output_file <path to your output file> \
 --vcf --compress_output gzip --sift s --polyphen s --numbers --symbol \
 --shift_3prime 1 --allele_number --refseq --total_length --no_stats --offline --cache \
---dir_cache </path/to/cache/107> --species "homo_sapiens" --assembly <GRCh37 or GRCh38> \
+--dir_cache </path/to/cache/107> --species "homo_sapiens" --assembly <GRCh38> \
 --fork <n_threads> --dont_skip --allow_non_variant --use_given_ref --exclude_predicted \
 --flag_pick_allele --plugin Grantham \
 --plugin SpliceAI,snv=<path/to/spliceai_scores.masked.snv.vcf.gz>,indel=</path/to/spliceai_scores.masked.indel.vcf.gz> \
---custom "<path/to/gnomad.total.sites.stripped.vcf.gz>,gnomAD,vcf,exact,0,AF,HN" \
---custom "<path/to/phyloP100way.bw>,phyloP,bigwig,exact,0" \
+--custom "<path/to/gnomad.total.sites.stripped.vcf.gz>,gnomAD,vcf,exact,0,AF" \
+--custom "<path/to/hg38.phyloP100way.bed.gz,phyloP,bed,exact,0" \
 --dir_plugins <path to your VEP plugin directory>
 ```
-**IMPORTANT: Ensure the right files are used based on GRCH37 or GRCH38!!!**
 
 Note: Certain arguments might not be needed if training/predicting without using all possible features offered by CAPICE.
 
@@ -333,19 +320,17 @@ To convert a `.pickle.dat` model to `.ubj`/`.json`, one can do the following (us
    - `sys.argv[1]`: the input `.pickle.dat` file
    - `sys.argv[2]`: the output `.json` file
 4. Adjust `CAPICE_version` within the new model file to that of the new release (major version should match!). 
-   It is advisable however to ensure that the filename still contains the version it was trained on (f.e. `v4.0.0_grch37_v5.0.0-compatibility.json`) to ensure no confusion will exist on which version it was actually trained.
+   It is advisable however to ensure that the filename still contains the version it was trained on (e.g. `v4.0.0_grch38_v5.0.0-compatibility.json`) to ensure no confusion will exist on which version it was actually trained.
 
 Do note that we recommend using a model trained on a specific major version instead, as other breaking changes might be present as well! 
 
 ## Data sources
 ### GnomAD
 The gnomAD files can be generated through the following scripts (which also download the gnomAD files):
-- GRCH37: https://github.com/molgenis/vip/blob/main/utils/create_gnomad_GRCh37.sh
 - GRCH38: https://github.com/molgenis/vip/blob/main/utils/create_gnomad_GRCh38.sh
 
 ### PhyloP
 PhyloP resources can be downloaded from:
-- GRCh37: http://hgdownload.cse.ucsc.edu/goldenpath/hg19/phyloP100way/
 - GRCh38: http://hgdownload.cse.ucsc.edu/goldenpath/hg38/phyloP100way/
 
 ### SpliceAI

diff --git a/resources/predict_input.tsv.gz b/resources/predict_input.tsv.gz
diff --git a/resources/predict_input_raw.vcf.gz b/resources/predict_input_raw.vcf.gz
diff --git a/resources/test_input.vcf b/resources/test_input.vcf
@@ -1,13 +1,13 @@
 ##fileformat=VCFv4.0
-##reference=GRCh37/hg19
+##reference=GRCh38
 #CHROM	POS	ID	REF	ALT
-12	69747417	.	C	A
-17	41231346	.	G	T
-2	122288533	.	C	A
-11	118382645	.	G	T
-5	235382	.	G	A
-2	48026421	.	T	C
-5	90073785	.	C	T
-1	63114155	.	T	C
-2	179431764	.	G	A
-9	131250286	.	G	A
+chr12	69747417	.	C	A
+chr17	41231346	.	G	T
+chr2	122288533	.	C	A
+chr11	118382645	.	G	T
+chr5	235382	.	G	A
+chr2	48026421	.	T	C
+chr5	90073785	.	C	T
+chr1	63114155	.	T	C
+chr2	179431764	.	G	A
+chr9	131250286	.	G	A
diff --git a/resources/train_features.json b/resources/train_features.json
@@ -17,6 +17,5 @@
     "SpliceAI_pred_DS_DG": null,
     "SpliceAI_pred_DS_DL": null,
     "Grantham": null,
-    "phyloP": null,
-    "gnomAD_HN": null
+    "phyloP": null
 }
diff --git a/resources/train_input.tsv.gz b/resources/train_input.tsv.gz
diff --git a/resources/train_input_raw.vcf.gz b/resources/train_input_raw.vcf.gz
diff --git a/resources/train_test.tsv.gz b/resources/train_test.tsv.gz
diff --git a/src/molgenis/capice/__init__.py b/src/molgenis/capice/__init__.py
@@ -1 +1 @@
-__version__ = '5.1.1'
+__version__ = '5.1.2'
diff --git a/tests/capice/test_main_train.py b/tests/capice/test_main_train.py
@@ -30,7 +30,7 @@ def tearDown(self):
 
     def setUp(self):
         print('Performing test:')
-        train_file = os.path.join(_project_root_directory, 'resources', 'train_input.tsv.gz')
+        train_file = os.path.join(_project_root_directory, 'resources', 'train_test.tsv.gz')
         impute_json = os.path.join(_project_root_directory,
                                    'resources',
                                    'train_features.json')
@@ -106,8 +106,7 @@ def test_integration_reset_train_features(self):
             'is_NMD_transcript_variant', 'is_feature_elongation', 'is_feature_truncation',
             'SpliceAI_pred_DP_AG', 'SpliceAI_pred_DP_AL', 'SpliceAI_pred_DP_DG',
             'SpliceAI_pred_DP_DL', 'SpliceAI_pred_DS_AG', 'SpliceAI_pred_DS_AL',
-            'SpliceAI_pred_DS_DG', 'SpliceAI_pred_DS_DL', 'Type', 'Length', 'Grantham', 'phyloP',
-            'gnomAD_HN'
+            'SpliceAI_pred_DS_DG', 'SpliceAI_pred_DS_DL', 'Type', 'Length', 'Grantham', 'phyloP'
         ]
         self.assertSetEqual(set(observed), set(expected))
 

diff --git a/tests/resources/breakends.vcf.gz b/tests/resources/breakends.vcf.gz
diff --git a/tests/resources/breakends_vep.tsv.gz b/tests/resources/breakends_vep.tsv.gz
diff --git a/tests/resources/edge_cases.vcf.gz b/tests/resources/edge_cases.vcf.gz
diff --git a/tests/resources/edge_cases_vep.tsv.gz b/tests/resources/edge_cases_vep.tsv.gz
diff --git a/tests/resources/symbolic_alleles.vcf.gz b/tests/resources/symbolic_alleles.vcf.gz
diff --git a/tests/resources/symbolic_alleles_vep.tsv.gz b/tests/resources/symbolic_alleles_vep.tsv.gz
diff --git a/tests/resources/xgb_booster_poc.ubj b/tests/resources/xgb_booster_poc.ubj