Merge pull request #105 from CCBR/use-tools

Use shared helper functions from ccbr_tools

bin/redirect

@@ -56,13 +56,11 @@ fi
 #   - snakemake
 # are in PATH
 if [[ $ISBIOWULF == true ]];then
-	# module purge
 	load_module_if_needed singularity
-	load_module_if_needed snakemake
+	load_module_if_needed snakemake/7
 elif [[ $ISFRCE == true ]];then
 # snakemake module on FRCE does not work as expected
 # use the conda installed version of snakemake instead
-	# module purge
 	load_module_if_needed load singularity
 	export PATH="/mnt/projects/CCBR-Pipelines/bin:$PATH"
 fi

pyproject.toml

@@ -38,6 +38,7 @@ classifiers = [
 requires-python = ">=3.11"
 dependencies = [
     "argparse",
+    "ccbr_tools@git+https://github.com/CCBR/Tools",
     "Click >= 8.1.3",
     "PySimpleGui < 5",
     "snakemake >= 7, < 8",

src/xavier/__main__.py

@@ -36,28 +36,20 @@
 """
 
 # Python standard library
-from __future__ import print_function
 import sys, os, subprocess, re, json, textwrap
 
 
 # 3rd party imports from pypi
 import argparse  # potential python3 3rd party package, added in python/3.5
+from ccbr_tools.pipeline.util import err, exists, fatal, permissions, require
+from ccbr_tools.pipeline.cache import check_cache
 
 # Local imports
 from .run import init, setup, bind, dryrun, runner, run
 from .shells import bash
 from .options import genome_options
-from .util import (
-    err,
-    exists,
-    fatal,
-    permissions,
-    check_cache,
-    require,
-    get_version,
-    get_genomes_list,
-)
 from .gui import launch_gui
+from .util import xavier_base, get_version
 
 __version__ = get_version()
 __email__ = "[email protected]"

src/xavier/gui.py

@@ -5,21 +5,23 @@
 import glob
 import PySimpleGUI as sg
 
-from .util import (
+from ccbr_tools.pipeline.util import (
     get_genomes_dict,
     get_tmp_dir,
-    xavier_base,
-    get_version,
     get_hpcname,
     check_python_version,
 )
-from .run import run_in_context
-from .cache import get_sif_cache_dir
+from ccbr_tools.pipeline.cache import get_sif_cache_dir
+from ccbr_tools.shell import exec_in_context
+
+from .util import xavier_base, get_version
+from .run import run
+
 
 def launch_gui(DEBUG=True):
     check_python_version()
     # get drop down genome options
-    jsons = get_genomes_dict()
+    jsons = get_genomes_dict(repo_base=xavier_base)
     genome_annotation_combinations = list(jsons.keys())
     genome_annotation_combinations.sort()
     if DEBUG:
@@ -165,7 +167,9 @@ def launch_gui(DEBUG=True):
     if DEBUG:
         print("layout is ready!")
 
-    window = sg.Window(f"XAVIER {get_version()}", layout, location=(0, 500), finalize=True)
+    window = sg.Window(
+        f"XAVIER {get_version()}", layout, location=(0, 500), finalize=True
+    )
     if DEBUG:
         print("window created!")
 
@@ -277,7 +281,11 @@ def launch_gui(DEBUG=True):
                 input=list(glob.glob(os.path.join(values["-INDIR-"], "*.fastq.gz"))),
                 output=output_dir,
                 genome=genome,
-                targets=values["-TARGETS-"] if values["-TARGETS-"] else xavier_base('resources', 'Agilent_SSv7_allExons_hg38.bed'), # TODO should this be part of the genome config file?
+                targets=values["-TARGETS-"]
+                if values["-TARGETS-"]
+                else xavier_base(
+                    "resources", "Agilent_SSv7_allExons_hg38.bed"
+                ),  # TODO should this be part of the genome config file?
                 mode="slurm",
                 job_name="pl:xavier",
                 callers=["mutect2", "mutect", "strelka", "vardict", "varscan"],
@@ -292,9 +300,9 @@ def launch_gui(DEBUG=True):
                 tmp_dir=get_tmp_dir(None, output_dir),
                 threads=2,
             )
-            allout_init = run_in_context(run_args)
+            allout_init = exec_in_context(run, run_args)
             run_args.runmode = "dryrun"
-            allout_dryrun = run_in_context(run_args)
+            allout_dryrun = exec_in_context(run, run_args)
             allout = "\n".join([allout_init, allout_dryrun])
             if DEBUG:
                 print(allout)
@@ -308,7 +316,7 @@ def launch_gui(DEBUG=True):
             )
             if ch == "Yes":
                 run_args.runmode = "run"
-                allout = run_in_context(run_args)
+                allout = exec_in_context(run, run_args)
                 sg.popup_scrolled(
                     allout,
                     title="Slurmrun:STDOUT/STDERR",
@@ -344,6 +352,7 @@ def launch_gui(DEBUG=True):
                 continue
     window.close()
 
+
 def copy_to_clipboard(string):
     r = Tk()
     r.withdraw()

src/xavier/options.py

@@ -1,13 +1,8 @@
 #!/usr/bin/env python3
 # -*- coding: UTF-8 -*-
 
-# Python standard library
-from __future__ import print_function
-
-# Local imports
-from .util import permissions
-
 import os
+from ccbr_tools.pipeline.util import permissions
 
 
 def genome_options(parser, user_option, prebuilt):
@@ -45,7 +40,3 @@ def genome_options(parser, user_option, prebuilt):
         )
 
     return user_option
-
-
-if __name__ == "__main__":
-    pass

src/xavier/run.py

@@ -11,20 +11,20 @@
 import shutil
 import sys
 import subprocess
-
-# Local imports
-from .util import (
+from ccbr_tools.pipeline.util import (
     git_commit_hash,
     join_jsons,
     fatal,
     which,
     exists,
     err,
-    get_version,
-    xavier_base,
     require,
     get_hpcname,
 )
+from ccbr_tools.pipeline.cache import image_cache
+
+# Local imports
+from .util import get_version, xavier_base
 
 
 def run(sub_args):
@@ -38,7 +38,7 @@ def run(sub_args):
     # Step 0. Check for required dependencies
     # The pipelines has only two requirements:
     # snakemake and singularity
-    require(["snakemake", "singularity"], ["snakemake", "singularity"])
+    require(["snakemake", "singularity"], ["snakemake/7", "singularity"])
 
     # Optional Step. Initialize working directory,
     # copy over required resources to run
@@ -360,7 +360,7 @@ def setup(sub_args, repo_path, output_path, create_nidap_folder_YN="no", links=[
 
     # Resolves if an image needs to be pulled from an OCI registry or
     # a local SIF generated from the rna-seek cache subcommand exists
-    config = image_cache(sub_args, config, repo_path)
+    config = image_cache(sub_args, config)
 
     # Add other cli collected info
     config["project"]["annotation"] = sub_args.genome
@@ -630,49 +630,6 @@ def add_rawdata_information(sub_args, config, ifiles):
     return config
 
 
-def image_cache(sub_args, config, repo_path):
-    """Adds Docker Image URIs, or SIF paths to config if singularity cache option is provided.
-    If singularity cache option is provided and a local SIF does not exist, a warning is
-    displayed and the image will be pulled from URI in 'config/containers/images.json'.
-    @param sub_args <parser.parse_args() object>:
-        Parsed arguments for run sub-command
-    @params config <file>:
-        Docker Image config file
-    @param repo_path <str>:
-        Path to RNA-seek source code and its templates
-    @return config <dict>:
-         Updated config dictionary containing user information (username and home directory)
-    """
-    images = os.path.join(repo_path, "config", "containers", "images.json")
-
-    # Read in config for docker image uris
-    with open(images, "r") as fh:
-        data = json.load(fh)
-    # Check if local sif exists
-    for image, uri in data["images"].items():
-        if sub_args.sif_cache:
-            sif = os.path.join(
-                sub_args.sif_cache,
-                "{}.sif".format(os.path.basename(uri).replace(":", "_")),
-            )
-            if not exists(sif):
-                # If local sif does not exist on in cache, print warning
-                # and default to pulling from URI in config/containers/images.json
-                print(
-                    'Warning: Local image "{}" does not exist in singularity cache'.format(
-                        sif
-                    ),
-                    file=sys.stderr,
-                )
-            else:
-                # Change pointer to image from Registry URI to local SIF
-                data["images"][image] = sif
-
-    config.update(data)
-
-    return config
-
-
 def get_nends(ifiles):
     """Determines whether the dataset is paired-end or single-end.
     If paired-end data, checks to see if both mates (R1 and R2) are present for each sample.
@@ -969,13 +926,3 @@ def runner(
             )
 
     return masterjob
-
-
-def run_in_context(args):
-    """Execute the run function in a context manager to capture stdout/stderr"""
-    with contextlib.redirect_stdout(io.StringIO()) as out_f, contextlib.redirect_stderr(
-        io.StringIO()
-    ) as err_f:
-        run(args)
-        allout = out_f.getvalue() + "\n" + err_f.getvalue()
-    return allout

src/xavier/shells.py

@@ -8,7 +8,7 @@
 import subprocess
 
 # Local imports
-from .util import err
+from ccbr_tools.pipeline.util import err
 
 
 def set_options(strict):
@@ -65,9 +65,3 @@ def bash(
         )
 
     return exitcode
-
-
-if __name__ == "__main__":
-    # Tests
-    bash("ls -la /home/")
-    bash("ls -la /fake/path")