Fix demultiplex tests, tidy test suites, add dummy fastq for use in t…

…esting

test/conftest.py

@@ -1,14 +1,14 @@
-#!/usr/bin/python3
 """
 Variables used across test modules, including the setup and teardown fixture
 that is run before and after every test
 """
 import os
+import re
 import shutil
 import pytest
 import tarfile
 import logging
-
+from shutil import copy
 # sys.path.append("..")
 from ad_logger import ad_logger
 from toolbox import toolbox
@@ -78,12 +78,12 @@ def create_logdirs():
     """
     Create temporary log directories for testing purposes
     """
-    os.makedirs(temp_log_dir)
+    os.makedirs(temp_log_dir, exist_ok=True)
     rf_obj = toolbox.RunfolderObject("TEST_FOLDER", ad_config.TIMESTAMP)
     for logfile in rf_obj.logfiles_config.values():
         parent_dir = os.path.dirname(logfile)
         if not os.path.isdir(parent_dir):
-            os.makedirs(parent_dir)
+            os.makedirs(parent_dir, exist_ok=True)
 
 
 def patch_toolbox(monkeypatch):
@@ -104,21 +104,35 @@ def run_before_and_after_session():
     """
     # Create temporary dirs for testing
     os.makedirs(
-        test_data_dir_unzipped
+        test_data_dir_unzipped, exist_ok=True
     )  # Holds the unzipped data to copy from for each test
 
     for tar in data_tars:
         with tarfile.open(tar["src"], "r:gz") as open_tar:
             open_tar.extractall(path=tar["dest"])
     for destination in to_copy_interop_to:
         shutil.copytree(os.path.join(test_data_dir_unzipped, "InterOp"), destination)
-    yield
+
+    test_data_unzipped = os.path.join(test_data_dir_unzipped, "demultiplex_test_files", "test_runfolders")
+
+    directories = [
+        os.path.join(test_data_unzipped, d)
+        for d in os.listdir(test_data_unzipped)
+        if os.path.isdir(os.path.join(test_data_unzipped, d))
+    ]
+    dummy_fastq = os.path.join(test_data_dir, "dummy_fastq.gz")
+
+    for directory in directories:
+        if re.match(".*999999_.*", directory):
+            fastqs_dir = os.path.join(test_data_unzipped, directory, "Data", "Intensities", "BaseCalls/")
+            os.makedirs(fastqs_dir, exist_ok=True)
+            copy(dummy_fastq, fastqs_dir)
+    yield  # Where the testing happens
     for to_remove in [test_data_dir_unzipped, test_data_temp]:
         if os.path.isdir(to_remove):
             shutil.rmtree(to_remove)
 
 
-# TODO fix patching of script loggers as this is not set up correctly !!
 @pytest.fixture(scope="function", autouse=True)
 def run_before_and_after_tests(monkeypatch):
     """

test/test_ad_email.py

@@ -1,10 +1,7 @@
-""" ad_email.py pytest unit tests
+""" ad_email.py pytest unit tests. The test suite is currently incomplete
 
 N.B. test_email_sending_success() will only pass when running on the
 workstation where the required auth details are stored
-
-# TODO write the below tests which are currently missing or incomplete:
-- AdEmail.generate_email_html
 """
 
 import pytest
@@ -14,6 +11,8 @@
 
 logger_obj = logger_obj
 
+# TODO finish this test suite as it is currently incomplete
+
 
 class TestAdEmail:
     """
@@ -47,9 +46,6 @@ def email_recipients(self):
             ],
         ]
 
-    # TODO write test for generate_email_html
-    # def test_generate_email_html():
-
     def test_send_email_success(
         self,
         logger_obj,

test/test_ad_logger.py

@@ -1,62 +1,11 @@
-#!/usr/bin/python3
-""" ad_logger.py pytest unit tests
-
-# TODO write the following unit tests which are currently missing or incomplete:
-- shutdown_logs
-- SensitiveFormatter
-    - format
-- AdLogger
-    - get_logger
-    - _get_file_handler
-    - _get_logging_formatter
-    - _get_syslog_handler
-    - _get_stream_handler
+""" ad_logger.py pytest unit tests. The test suite is currently incomplete
 """
 import pytest
 from toolbox import toolbox
 from ad_logger import ad_logger
 from config import ad_config
 
-# import logging
-
-# No logging disabled for this test as we are testing logging
-# No patching required
-
-# TODO add tests for SensitiveFormatter class
-# TODO add test for shutdown_logs
-# TODO add test that checks that streamhandler, filehandler and syslog handler are all
-# added as expected
-
-
-# TODO write test for shutdown_logs()
-# def test_shutdown_logs():
-#     """"""
-
-#     ad_logger.shutdown_logs(logger)
-# assert not logger.info(
-#     "Test log message. Logger %s",
-#     logger.name,
-# )
-
-# TODO write tests for SensitiveFormatter class
-# class TestSensitiveFormatter:
-#    """
-#    Tests for the SensitiveFormatter class
-#    """
-
-
-# @pytest.fixture(scope="function", autouse=True)
-# def setup(monkeypatch):
-#     """
-#     """
-#     #  Re-enable logging as it is required for assertions
-#     logging.disable(logging.NOTSET)
-#     # Remove testfiles dir containing test runfolders as we don't need these files
-#     # Apply patches required for test_ad_logger script. These point the paths to the
-#     # temporary locations:
-#     #     - Test logfiles in the temp logfiles dir and within the temp runfolder dirs
-#     monkeypatch.setattr(toolbox.ad_config, "RUNFOLDERS", conftest.temp_runfolderdir)
-#     monkeypatch.setattr(toolbox.ad_config, "AD_LOGDIR", conftest.temp_log_dir)
+# TODO finish this test suite as it is currently incomplete
 
 
 class TestRunfolderLoggers:
@@ -99,8 +48,4 @@ def test_get_loggers(self, logfiles_config, caplog):
             )
             assert loggers[logger_name].name in caplog.text
 
-    # TODO write tests for AdLogger class
-    # class TestAdLogger:
-    """
-    Tests for the AdLogger class
-    """
+