jovian

#!/bin/bash
############################################################################################################
### Authors:                                                                                             ###
###     Dennis Schmitz, Sam Nooij, Florian Zwagemaker, Robert Verhagen,                                  ###
###     Jeroen Cremer, Thierry Janssens, Mark Kroon, Erwin van Wieringen,                                ###
###     Annelies Kroneman, Harry Vennema, Marion Koopmans                                                ###
### Organization:                                                                                        ###
###     Rijksinstituut voor Volksgezondheid en Milieu (RIVM)                                             ###
###     Dutch Public Health institute (https://www.rivm.nl/en)                                           ###
### Department:                                                                                          ###
###     Virology - Emerging and Endemic Viruses (EEV)                                                    ###
###     Virology - Viruses of the Vaccination Program (VVP)                                              ###
###     Bacteriology - Bacterial and Parasitic Diagnostics (BPD)                                         ###
### Date and license:                                                                                    ###
###     23-08-2018, AGPL3 license                                                                        ###
### Homepage containing documentation, examples and a changelog:                                         ###
###     https://github.com/DennisSchmitz/Jovian                                                          ###
### Funding:                                                                                             ###
###     This project/research has received funding from the European Union’s                             ###
###     Horizon 2020 research and innovation programme under grant agreement                             ###
###     No. 643476. and the Dutch working group on molecular diagnostics (WMDI).                         ###
### Automation:                                                                                          ###
###    iRODS automatically executes this workflow for all "vir-ngs" labelled                             ###
###    Illumina sequencing runs. See the internal Gitlab repo for the wrapper                            ###
###    with additional information.                                                                      ###
############################################################################################################

# shellcheck disable=SC1091
#set -euo pipefail
#set -v
#set -x

#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#@ load in standard vars

set -o allexport
[[ -f .env ]] && source .env
source bin/includes/functions
eval "$(parse_yaml config/pipeline_parameters.yaml "params_")"
eval "$(parse_yaml config/config.yaml "configuration_")"
set +o allexport

#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#@ Parse the jovian commandline arguments.
#@ If the given arguments are not part of Jovian, then Jovian will send them to Snakemake
#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

shopt -s nocasematch

POSITIONAL=()
while [[ $# -gt 0 ]]
do
    key="$1"
    case $key in
        im|illumina-metagenomics|illumina-meta|ilm-meta)
        workflow="ILM_META"
        export workflow
        shift # Next
        ;;
        ir|illumina-reference|illumina-ref|ilm-ref)
        workflow="ILM_REF"
        export workflow
        shift # Next
        ;;
        nr|nanopore-reference|nanopore-ref|nano-ref|ont-ref)
        workflow="NANO_REF"
        export workflow
        shift # Next
        ;;
        -i|--input)
        INPUT_DIR="$2"
        export INPUT_DIR
        shift # Next
        shift # Next
        ;;
        -m|--mode)
        JOVIAN_MODE="$2"
        export JOVIAN_MODE
        shift # Next
        shift # Next
        ;;
        -h|--help)
        JOVIAN_HELP="TRUE"
        export JOVIAN_HELP
        shift # Next
        ;;
        --find-amplicon-coverage)
        inputfile="$2"
        FIND_AMP_STATS="TRUE"
        export FIND_AMP_STATS
        shift
        shift
        ;;
        --db-reset)
        RESET_DB="TRUE"
        export RESET_DB
        shift # Next
        ;;
        -sh|--snakemake-help)
        SNAKEMAKE_HELP="TRUE"
        export SNAKEMAKE_HELP
        shift # Next
        ;;
        -ic|--install-dependencies)
        INSTALL_CONDA_ENVS="TRUE"
        export INSTALL_CONDA_ENVS
        shift # Next
        ;;
        -id|--install-databases)
        INSTALL_DB="TRUE"
        export INSTALL_DB
        shift # Next
        ;;
        --clean)
        CLEAN="TRUE"
        export CLEAN
        shift # Next
        ;;
        --archive)
        ARCHIVE="TRUE"
        export ARCHIVE
        shift # Next
        ;;
        -v|--version)
        SHOW_VERSION="TRUE"
        export SHOW_VERSION
        shift # Next
        ;;
        --start-nginx)
        START_NGINX="TRUE"
        export START_NGINX
        shift # Next
        ;;
        --stop-nginx)
        STOP_NGINX="TRUE"
        export STOP_NGINX
        shift # Next
        ;;
        --rebuild-archive)
        REBUILD_ARCHIVE="TRUE"
        export REBUILD_ARCHIVE
        shift # Next
        ;;
        --configure-jupyter)
        CONFIG_JUP="TRUE"
        export CONFIG_JUP
        shift # Next
        ;;
        --start-jupyter)
        START_JUPYTER="TRUE"
        export START_JUPYTER
        shift # Next
        ;;
        -vt|--virus-typing)
        VIRUS_TYPING="TRUE"
        WHICH_TT="$2"
        export VIRUS_TYPING
        export WHICH_TT
        shift # Next
        shift # Next
        ;;
        -vt-force|--virus-typing-force)
        VIRUS_TYPING="TRUE"
        FORCE_OVERWRITE_TT="TRUE"
        WHICH_TT="$2"
        export VIRUS_TYPING
        export FORCE_OVERWRITE_TT
        export WHICH_TT
        shift # Next
        shift # Next
        ;;
        -vt-help|--virus-typing-help)
        VIRUS_TYPING="TRUE"
        VIRUS_TYPING_HELP="TRUE"
        export VIRUS_TYPING
        export VIRUS_TYPING_HELP
        shift # Next
        ;;
        --make-sample-sheet)
        MAKE_SAMPLE_SHEET="TRUE"
        export MAKE_SAMPLE_SHEET
        shift # Next
        ;;
        -y)
        SKIP_CONFIRMATION="TRUE"
        export SKIP_CONFIRMATION
        shift # Next
        ;;
        -u|--unlock)
        SNAKEMAKE_UNLOCK="TRUE"
        export SNAKEMAKE_UNLOCK
        shift # Next
        ;;
        --update)
        JOVIAN_UPDATE="TRUE"
        CHANGE_VERSION="$2"
        export JOVIAN_UPDATE
        export CHANGE_VERSION
        shift # Next
        shift # Next
        ;;
        -cq|--cluster-queue)
        QUEUE_NAME_OVERRIDE="TRUE"
        QUEUE_NAME_REPLACEMENT="$2"
        export QUEUE_NAME_OVERRIDE
        export QUEUE_NAME_REPLACEMENT
        shift # Next
        shift # Next
        ;;
        -ref|--reference)
        REFERENCE_FASTA="$2"
        export REFERENCE_FASTA
        shift # Next
        shift # Next
        ;;
        -pr|--primers)
        PRIMER_FASTA="$2"
        export PRIMER_FASTA
        shift # Next
        shift # Next
        ;;
        *) # Any other option
        POSITIONAL+=("$1") # save in array
        shift # Next
        ;;
    esac
done
set -- "${POSITIONAL[@]:-}" #> Restores the positional arguments 
                            #> (i.e. without the case arguments above) which then can be called via `$@` or `$[0-9]` etc.
                            #> These parameters are send to Snakemake.
shopt -u nocasematch

#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#@ 
#@ First basic arguments that do not require Jovian to be installed
#@
#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #


###> Show Jovian version
if [ "${SHOW_VERSION}" == "TRUE" ]; then
    bash bin/includes/Version
    exit 0
fi

###> Remove all Jovian output
if [ "${CLEAN:-}" == "TRUE" ]; then
    bash bin/includes/Clean
    exit 0
fi

###> Print Jovian help message
if [ "${JOVIAN_HELP:-}" == "TRUE" ]; then
    bash bin/includes/help
    exit 0
fi

if [ "${RESET_DB}" == "TRUE" ]; then
    rm ~/.jovian_installchoice_db
    if ! bash bin/includes/Pre_install_checker; then
        exit 1
    fi
    bash bin/includes/Userconfig_DB
    exit
fi

###> Change the installed Jovian version to the latest release or to a specified verison
if [ "${JOVIAN_UPDATE}" == "TRUE" ]; then
    if [ -z "${CHANGE_VERSION}" ]; then
        bash bin/includes/selfupdate master
        exit 0
    else
        bash bin/includes/selfupdate "${CHANGE_VERSION}"
        exit
    fi
fi

#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#@ 
#@ Jovian Installation block
#@
#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

###> Pre-flight check: Assess availability of required files, conda and Jovian_master environment
if ! bash bin/includes/Pre_install_checker; then
    exit 1
fi

###> Installer for miniconda.
###> Runs if the command "conda" is not recognised by the host AND if the "~/UserBinaries/Miniconda3" path is non-existant
if ! bash bin/includes/Install_miniconda; then
    exit 1
fi


set -o allexport
source "${HOME}"/.jovianrc
set +o allexport

###> Installer for Jovian_master environment.
###> Runs if the master environment is not active (in path) and is not installed either. Activates environment on finish
if ! bash bin/includes/Install_jovian-master; then
    exit 1
else
    set -o allexport
    conda activate "${MASTER_NAME}"
    set +o allexport
    echo -e "Successfully activated Jovian_master environment"
fi


###> Install databases, as specified via CLI argument '-id'
if [ "${INSTALL_DB:-}" == "TRUE" ]; then
    bash bin/includes/Databases
    exit 0
fi

###> Fill in the global location of the databases.
bash bin/includes/Userconfig_DB

###> Fill in the global computing-mode configuration
if ! bash bin/includes/Userconfig_CM; then
    exit
fi

#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#@ 
#@ End of Jovian installation.
#@ From this point onwards you can safely assume Conda is installed
#@ The Jovian_master environment is activated for this shell session and all child-shells
#@
#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

###> Check the snakemake version
env_snakever=$(snakemake -v)

 if [ !  "$(printf '%s\n' "$req_snakever" "$env_snakever" | sort -V | head -n1)" = "$req_snakever" ]; then 
        echo -e "The snakemake version in your \"${MASTER_NAME}\" environment does not meet the required version of snakemake $req_snakever or higher"
        echo -e "Please make sure the \"${MASTER_NAME}\" environment uses the correct dependencies by typing: \e[7m bash jovian --update \e[0m"
        exit
 fi

###> Archiving of relevant data in a single tar.gz file
if [ "${ARCHIVE:-}" == "TRUE" ]; then
    bash bin/includes/Archive
    exit 0
fi

###> Rebuild archived Jovian results in order to view results and data.
#TODO the syntax below doesn't work if there are multiple archives. should really fix at some point.
if [ "${REBUILD_ARCHIVE}" == "TRUE" ]; then
    if [ -e archive_*.tar.gz ]; then
        bin/includes/Rebuild_archive
    exit 0
    else
        echo -e "There's no Jovian-archive present."
        exit 0
    fi
fi

###> Print Snakemake help
if [ "${SNAKEMAKE_HELP:-}" == "TRUE" ]; then
    line
    snakemake --help
    exit 0
fi

###> find average depth of coverage for amplicons
if [ "${FIND_AMP_STATS}" == "TRUE" ]; then
    bash bin/includes/amplicon_cov_wrapper $inputfile
    exit
fi
#@ # # # # # #
#@ virus typing part

###> Main controller for the virus typing block, as specified via CLI argument '--virus-typing'
if [ "${VIRUS_TYPING:-}" == "TRUE" ]; then
    if [ "${VIRUS_TYPING_HELP:-}" == "TRUE" ]; then
        bash bin/scripts/virus_typing.sh --help
        exit 0
    elif [ "${FORCE_OVERWRITE_TT:-}" == "TRUE" ]; then
        
        # Check if the URL's are accessible (sometimes they are overloaded/down for maintenance/crashed)
        if ! bash bin/includes/Typingtool_check "${WHICH_TT}"; then
            exit 1
        fi

        bash bin/scripts/virus_typing.sh "${WHICH_TT}" --force
        exit 0
    else

        # Check if the URL's are accessible (sometimes they are overloaded/down for maintenance/crashed)
        if ! bash bin/includes/Typingtool_check "${WHICH_TT}"; then
            exit 1
        fi
        
        bash bin/scripts/virus_typing.sh "${WHICH_TT}"
        exit 0
    fi
fi



###> Make a .ncbirc file based on the database information in ~/.jovian_installchoice_db
bash bin/includes/Make_ncbirc


###> Installation of Jovian specific conda environments on user request with "bash jovian -ic"
if ! bash bin/includes/Install_conda_envs; then
    exit 0
fi

###> Start and/or stop nginx
if [ "${START_NGINX}" == "TRUE" ]; then
    (bin/includes/Start_nginx start)
    exit 0
fi
if [ "${STOP_NGINX}" == "TRUE" ]; then
    nginx -s quit
    echo "nginx has been stopped"
    exit 0
fi

###> Auto configure the jupyter user profile and prepare it for use without manually configuring it
if [ "${CONFIG_JUP}" == "TRUE" ]; then
    bash bin/includes/Config_jupyter
    exit 0
fi

###> Easy command to start jupyter notebooks
if [ "${START_JUPYTER}" == "TRUE" ]; then
    bash bin/includes/Start_jupyter
    exit 0
fi

###> Detect mode and/or switch mode on request of the user
if ! bash bin/includes/Mode; then
    exit 1
fi

###> catch missing workflow statement
######! Make a dedicated error catcher for this one. Also to actually capture more missing statements than just the workflow.
if [ $workflow == "NONE" ]; then
    echo "No workflow was specified."
    echo "The following workflows are supported: ${supp_workflow[*]}"
    echo "Please see the documentation and try again"
    exit
fi


if [ "${workflow}" == "ILM_META" ]; then
    nicename="Illumina-metagenomics"
elif [ "${workflow}" == "ILM_REF" ]; then
    nicename="Illumina-reference"
elif [ "${workflow}" == "NANO_REF" ]; then
    nicename="Nanopore-reference"
fi

###> Set the variables in the pipeline_parameters.yaml file
ref_parameter="reference_file: ${REFERENCE_FASTA}"
prm_parameter="primer_file: ${PRIMER_FASTA}"
sed -i "s~.*reference_file:.*~${ref_parameter}~g" "${PROFILE}"/pipeline_parameters.yaml
sed -i "s~.*primer_file:.*~${prm_parameter}~g" "${PROFILE}"/pipeline_parameters.yaml


###> Build the sample_sheet.yaml
if ! bash bin/includes/Make_samplesheet; then
    exit 1
fi

###? Failsafe 1 for the sample sheet
if [ -e sample_sheet.yaml ]; then
    echo -e "Succesfully generated the sample sheet"
else
    echo -e "sample_sheet.yaml could not be found"
    echo -e "This also means that Jovian was unable to generate a new sample sheet for you"
    echo -e "Please inspect the input directory (${INPUT_DIR}) and make sure the right files are present"
    exit 1
fi

###> Make the sample sheet and then quit Jovian for manual running and debugging of the pipeline
if [ "${MAKE_SAMPLE_SHEET}" == "TRUE" ] && [ ! -s config/variables.yaml ]; then #? If config/variables.yaml does not exist or is empty AND manual make sample sheet is TRUE...
    echo -e "Jovian_run:\n    identifier: ${UNIQUE_ID}" > config/variables.yaml
    echo -e "Server_host:\n    hostname: http://${HOSTNAME}" >> config/variables.yaml
    echo -e "The variables file has been created, you can now run the Jovian snakefile manually"
    exit 0
elif [ "${MAKE_SAMPLE_SHEET}" == "TRUE" ] && [ -s config/variables.yaml ]; then
    echo -e "Variable file not generated, it already existed. You are now able to run the Jovian snakefile manually."
    exit 0
fi

###> Here the UNIQUE_ID and HOSTNAME variables are actually set
if [ ! -s config/variables.yaml ]; then #? If config/variables.yaml does not exist or is empty...
    echo -e "Jovian_run:\n    identifier: ${UNIQUE_ID}" > config/variables.yaml
    echo -e "Server_host:\n    hostname: http://${HOSTNAME}" >> config/variables.yaml
    eval $(parse_yaml config/variables.yaml "config_")
else
    eval $(parse_yaml config/variables.yaml "config_")
fi

###> Unlock snakemake if, for some reason, snakemake was unable to unlock the working directory by itself
if [ "${SNAKEMAKE_UNLOCK}" == "TRUE" ]; then
    printf "\nUnlocking working directory...\n"
    if [ "${workflow}" == "ILM_META" ]; then
        snakemake -s bin/Illumina_Meta.smk --profile "${PROFILE}" --unlock
    fi
    if [ "${workflow}" == "ILM_REF" ]; then
        snakemake -s bin/Illumina_vir_Ref.smk --profile "${PROFILE}" --unlock
    fi
    if [ "${workflow}" == "NANO_REF" ]; then
        snakemake -s bin/Nano_vir_Ref.smk --profile "${PROFILE}" --unlock
    fi
    printf "\nDone.\n"
    exit 0
fi

#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#@ Last Preflight checkers here.

###> Check databases locations, exit if something is missing
if ! bash bin/includes/Preflight_check_databases; then
    exit 1
fi
###> Check the reads and change pipeline parameters acccordingly
if ! bash bin/includes/Preflight_readlength-counter; then
exit 1
fi

###> Check the reference fasta file if given to see whether or not the file is actually valid
if ! bash bin/includes/Reference_validation; then
exit 1
fi

##> Check the primer fasta file and see whether or not the file is actually valid
if ! bash bin/includes/Primer_validation; then
exit 1
fi

ready_for_start
#@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

###> Core Jovian workflow snakemake command with checkers for required files.

#@ Illumina-meta workflow
if [ "${workflow}" == "ILM_META" ]; then
    echo -e "Starting the \"${nicename}\" workflow."
    set +ue #turn off bash strict mode because snakemake and conda don't work in strict mode
    snakemake -s bin/Illumina_Meta.smk --conda-frontend mamba --profile "${PROFILE}" ${@} && echo -e "\nUnique identifier for this Jovian run is: $config_Jovian_run_identifier "
    set -ue #turn bash strict mode back on
    exit 0
fi

#@ Illumina-Ref workflow
if [ "${workflow}" == "ILM_REF" ]; then
    echo -e "Starting the \"${nicename}\" workflow."
    set +ue #turn off bash strict mode because snakemake and conda don't work in strict mode
    snakemake -s bin/Illumina_vir_Ref.smk --conda-frontend mamba --profile "${PROFILE}" ${@} && echo -e "\nUnique identifier for this Jovian run is: $config_Jovian_run_identifier "
    set -ue #turn bash strict mode back on
    exit 0
fi

#@ Nanopore-ref workflow
if [ "${workflow}" == "NANO_REF" ]; then
    echo -e "Starting the \"${nicename}\" workflow."
    set +ue
    snakemake -s bin/Nano_vir_Ref.smk --conda-frontend mamba --profile "${PROFILE}" ${@} && echo -e "\nUnique identifier for this Jovian run is: $config_Jovian_run_identifier "
    set -ue
    exit 0
fi


exit 0