#49 - added pytest db dependency test + readmy

bedboss/bedboss.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import urllib.request
 from typing import NoReturn, Union, Dict
 import pypiper
 from argparse import Namespace
@@ -19,7 +18,12 @@
     BED_FOLDER_NAME,
     BIGBED_FOLDER_NAME,
 )
-from bedboss.utils import extract_file_name, standardize_genome_name, download_file
+from bedboss.utils import (
+    extract_file_name,
+    standardize_genome_name,
+    download_file,
+    check_db_connection,
+)
 from bedboss.exceptions import OpenSignalMatrixException
 from bedboss import __version__
 
@@ -29,16 +33,17 @@
 def get_osm_path(genome: str) -> Union[str, None]:
     """
     By providing genome name download Open Signal Matrix
+
     :param genome: genome assembly
     :return: path to the Open Signal Matrix
     """
     # TODO: add more osm
     _LOGGER.info(f"Getting Open Signal Matrix file path...")
-    if genome == "hg19":
+    if genome == "hg19" or genome == "GRCh37":
         osm_name = OS_HG19
-    elif genome == "hg38":
+    elif genome == "hg38" or genome == "GRCh38":
         osm_name = OS_HG38
-    elif genome == "mm10":
+    elif genome == "mm10" or genome == "GRCm38":
         osm_name = OS_MM10
     else:
         raise OpenSignalMatrixException(
@@ -76,6 +81,7 @@ def run_all(
 ) -> NoReturn:
     """
     Run bedboss: bedmaker, bedqc and bedstat.
+
     :param sample_name: Sample name [required]
     :param input_file: Input file [required]
     :param input_type: Input type [required] options: (bigwig|bedgraph|bed|bigbed|wig)
@@ -99,6 +105,10 @@ def run_all(
     :return: NoReturn
     """
     _LOGGER.warning(f"Unused arguments: {kwargs}")
+
+    if not check_db_connection(bedbase_config=bedbase_config):
+        raise Exception("Database connection failed. Exiting...")
+
     file_name = extract_file_name(input_file)
     genome = standardize_genome_name(genome)
 
@@ -153,6 +163,7 @@ def run_all(
 def main(test_args: dict = None) -> NoReturn:
     """
     Run pipeline that was specified in as positional argument.
+
     :param str test_args: one of the bedboss pipelines
     """
     parser = build_argparser()
@@ -174,10 +185,6 @@ def main(test_args: dict = None) -> NoReturn:
         multi=True,
         version=__version__,
     )
-
-    # Confirm everything is set up correctly
-    # TODO: add more checks to make sure everything we need is installed
-
     if args_dict["command"] == "all":
         run_all(pm=pm, **args_dict)
     elif args_dict["command"] == "make":

bedboss/utils.py

@@ -2,6 +2,7 @@
 import logging
 import requests
 import urllib
+from bbconf import BedBaseConf
 from typing import NoReturn
 
 
@@ -11,6 +12,7 @@
 def extract_file_name(file_path: str) -> str:
     """
     Extraction file name from file path
+
     :param file_path: full file path
     :return: file name without extension
     """
@@ -22,6 +24,7 @@ def extract_file_name(file_path: str) -> str:
 def standardize_genome_name(input_genome: str) -> str:
     """
     Standardizing user provided genome
+
     :param input_genome: standardize user provided genome, so bedboss know what genome
     we should use
     :return: genome name string
@@ -43,6 +46,7 @@ def standardize_genome_name(input_genome: str) -> str:
 def download_file(url: str, path: str) -> NoReturn:
     """
     Download file from the url to specific location
+
     :param url: URL of the file
     :param path: Local path with filename
     :return: NoReturn
@@ -55,3 +59,24 @@ def download_file(url: str, path: str) -> NoReturn:
     except Exception as e:
         _LOGGER.error(f"File download failed.")
         raise e
+
+
+def check_db_connection(bedbase_config: str) -> bool:
+    """
+    Check if the database connection is working
+
+    :param bedbase_config: path to the bedbase config file
+    :return: True if connection is successful, False otherwise
+    """
+    _LOGGER.info(f"Checking database connection...")
+    if not os.path.exists(bedbase_config):
+        raise FileNotFoundError(f"Bedbase config file {bedbase_config} was not found.")
+    else:
+        _LOGGER.info(f"Bedbase config file {bedbase_config} was found.")
+    try:
+        BedBaseConf(bedbase_config)
+        _LOGGER.info(f"Database connection is successful.")
+        return True
+    except Exception as e:
+        _LOGGER.error(f"Database connection failed. Error: {e}")
+        return False

test/README.md

@@ -0,0 +1,29 @@
+# How to setup the test environment
+
+## Check if all dependencies are installed or install them
+
+To check if all dependencies are installed run the following command:
+
+go to test folder in the project root directory and run:
+```bash
+bash ./bash_requirements_test.sh
+```
+
+## Set up database or use existing one by changing the config file. Use one of the following options:
+1) Open `bedbase_config_test.yaml` file in the `test/test_dependencies` folder, and change database credentials the one that you want to use.
+2) Create a new database and user with the credentials that are in the `bedbase_config_test.yaml` file. Credentials that
+are in the config file are:
+```text
+  host: localhost
+  port: 5432
+  password: docker
+  user: postgres
+  name: bedbase
+```
+
+### To create a new database and user with the credentials that are in the `bedbase_config_test.yaml` file, run the following commands:
+
+1) Go to `db_setup` directory and then run the following lines
+2) Build the docker: `docker build -t bedbase ./`
+3) Run the docker: `docker run --name bedbase -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=docker -p 5432:5432 -d bedbase`
+4) Start it: `docker start bedbase`

test/db_setup/Dockerfile

@@ -0,0 +1,4 @@
+FROM postgres
+ENV POSTGRES_USER postgres
+ENV POSTGRES_PASSWORD docker
+ENV POSTGRES_DB bedbase

test/test_bedboss.py

@@ -2,6 +2,7 @@
 import os
 import subprocess
 import pytest
+from bbconf import BedBaseConf
 
 FILE_DIR = os.path.dirname(os.path.realpath(__file__))
 HG19_CORRECT_DIR = os.path.join(FILE_DIR, "data", "bed", "hg19", "correct")
@@ -16,12 +17,25 @@
 
 def test_dependencies():
     # Make sure bedToBigBed etc is in your PATH.
+    print("Testing dependencies...")
     key = "PATH"
     value = os.getenv(key)
     test_dep_return_code = subprocess.run([DEPENDENCIES_TEST_SCRIPT], shell=True)
     assert 1 > test_dep_return_code.returncode
 
 
+def db_setup():
+    # Check if the database is setup
+    try:
+        BedBaseConf(BEDBASE_CONFIG)
+    except Exception as err:
+        return False
+    return True
+
+
+pytest_db_skip_reason = "Database is not set up... To run this test, set up the database. Go to test/README.md for more information."
+
+
 @pytest.mark.parametrize(
     "bedfile",
     [
@@ -63,6 +77,10 @@ def test_make(bedfile, tmpdir):
     assert os.path.isfile(os.path.join(tmpdir, "bigbed", "sample1.bigBed"))
 
 
+@pytest.mark.skipif(
+    not db_setup(),
+    reason=pytest_db_skip_reason,
+)
 class TestStat:
     @pytest.fixture(scope="session")
     def output_temp_dir(self, tmp_path_factory):
@@ -127,6 +145,10 @@ def test_check_file_exists(self, file, output_temp_dir):
         )
 
 
+@pytest.mark.skipif(
+    not db_setup(),
+    reason=pytest_db_skip_reason,
+)
 class TestAll:
     @pytest.fixture(scope="session")
     def output_temp_dir(self, tmp_path_factory):

test/test_dependencies/bedbase_config_test.yaml

@@ -8,7 +8,8 @@ database:
   port: 5432
   password: docker
   user: postgres
-  name: pep-db
+  name: bedbase
+  #name: pep-db
   dialect: postgresql
   driver: psycopg2
 server:
@@ -21,20 +22,3 @@ remotes:
   s3:
     prefix: s3://data.bedbase.org/
     description: S3 compatible path
-
-#database:
-#  name: pipestat-test
-#  user: postgres
-#  password: pipestat-password
-#  host: localhost
-#  dialect: postgresql
-#  driver: psycopg2
-#  port: 5432
-##path:
-##  pipeline_output_path: $BEDBASE_DATA_PATH/outputs
-##  bedstat_dir: bedstat_output
-##  bedbuncher_dir: bedbuncher_output
-##  remote_url_base: null
-#server:
-#  host: 0.0.0.0
-#  port: 8000