Merge pull request #74 from LUMC/release_1.2.0

Release 1.2.0

HISTORY.rst

@@ -7,6 +7,15 @@ Changelog
 .. NOTE: This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.2.0
+---------------------------
++ Giving a ``--basetemp`` directory that is within pytest's current working
+  directory will now raise an exception to prevent infinite recursive directory
+  copying.
++ The cleanup message is only displayed when pytest-workflow is used.
++ Added a ``--keep-workflow-wd-on-fail`` or ``--kwdof`` flag. Setting this flag
+  will make sure temporary directories are only deleted when all tests succeed.
+
 version 1.1.2
 ---------------------------
 + Fixed a bug where the program would hang indefinitely after a user input

README.rst

@@ -9,7 +9,7 @@ pytest-workflow
   :alt:
 
 .. image:: https://img.shields.io/conda/v/bioconda/pytest-workflow.svg
-  :target: https://https://bioconda.github.io/recipes/pytest-workflow/README.html
+  :target: https://bioconda.github.io/recipes/pytest-workflow/README.html
   :alt:
 
 .. image:: https://img.shields.io/pypi/pyversions/pytest-workflow.svg

docs/running_pytest_workflow.rst

@@ -8,11 +8,15 @@ Pytest will automatically gather files in the ``tests`` directory starting with
 
 The workflows are run automatically. Each workflow gets its own temporary
 directory to run. The ``stdout`` and ``stderr`` of the workflow command are
-also saved to this directory. The temporary directories are cleaned up after
-the tests are completed. If you wish to inspect the output of a failing
-workflow you can use the ``--kwd`` or ``--keep-workflow-wd`` flag to disable
-cleanup. This will also make sure the logs of the pipeline are not deleted. The
-``--keep-workflow-wd`` flag is highly recommended when debugging pipelines.
+also saved to this directory to ``log.out`` and ``log.err`` respectively.
+The temporary directories are cleaned up after the tests are completed.
+If you wish to inspect the output of a failing
+workflow you can use the ``--keep-workflow-wd`` or ``--kwd`` flag to disable
+cleanup. This will also make sure the logs of the pipeline are not deleted.
+If you only want to keep directories when one or more tests fail you can use
+the ``--keep-workflow-wd-on-fail`` or ``--kwdof`` flag.
+``--keep-workflow-wd-on-fail`` will keep all temporary directories, even from
+workflows that have succeeded.
 
 If you wish to change the temporary directory in which the workflows are run
 use ``--basetemp <dir>`` to change pytest's base temp directory.

setup.py

@@ -21,7 +21,7 @@
 
 setup(
     name="pytest-workflow",
-    version="1.1.2",
+    version="1.2.0",
     description="A pytest plugin for configuring workflow/pipeline tests "
                 "using YAML files",
     author="Leiden University Medical Center",

src/pytest_workflow/__init__.py

@@ -15,6 +15,9 @@
 # along with pytest-workflow.  If not, see <https://www.gnu.org/licenses/
 
 import re
+import shutil
+from pathlib import Path
+from typing import List
 
 
 # This function was created to ensure the same conversion is used throughout
@@ -27,3 +30,8 @@ def replace_whitespace(string: str, replace_with: str = '_') -> str:
     :return: The string with whitespace converted.
     """
     return re.sub(r'\s+', replace_with, string)
+
+
+def rm_dirs(directories: List[Path]):
+    for directory in directories:
+        shutil.rmtree(str(directory))

src/pytest_workflow/plugin.py

@@ -30,7 +30,7 @@
 
 import yaml
 
-from . import replace_whitespace
+from . import replace_whitespace, rm_dirs
 from .content_tests import ContentTestCollector
 from .file_tests import FileTestCollector
 from .schema import WorkflowTest, workflow_tests_from_schema
@@ -45,6 +45,13 @@ def pytest_addoption(parser: PytestParser):
              "debugging purposes. This also triggers saving of stdout and "
              "stderr in the workflow directory",
         dest="keep_workflow_wd")
+    parser.addoption(
+        "--kwdof", "--keep-workflow-wd-on-fail",
+        action="store_true",
+        help="Similar to --keep-workflow-wd, but only keeps the temporary "
+             "directories if there are test failures. On success all "
+             "directories are deleted.",
+        dest="keep_workflow_wd_on_fail")
     parser.addoption(
         "--wt", "--workflow-threads",
         dest="workflow_threads",
@@ -126,6 +133,15 @@ def pytest_configure(config: PytestConfig):
     workflow_temp_dir = (
         Path(basetemp) if basetemp is not None
         else Path(tempfile.mkdtemp(prefix="pytest_workflow_")))
+
+    rootdir = Path(str(config.rootdir))
+    # Raise an error if the workflow temporary directory of the rootdir
+    # (pytest's CWD). This will lead to infinite looping and copying.
+    if str(workflow_temp_dir.absolute()).startswith(str(rootdir.absolute())):
+        raise ValueError("'{0}' is a subdirectory of '{1}'. Please select a "
+                         "--basetemp that is not in pytest's current working "
+                         "directory.".format(workflow_temp_dir, rootdir))
+
     setattr(config, "workflow_temp_dir", workflow_temp_dir)
 
 
@@ -178,13 +194,26 @@ def pytest_runtestloop(session: pytest.Session):
     )
 
 
-def pytest_sessionfinish(session: pytest.Session):
-    if not session.config.getoption("keep_workflow_wd"):
-        # The newline is needed otherwise everything looks ugly.
-        print("\nRemoving temporary directories and logs. Use '--kwd' or "
+def pytest_sessionfinish(session: pytest.Session, exitstatus: int):
+    # No cleanup needed if we keep workflow directories
+    # Or if there are no directories to cleanup. (I.e. pytest-workflow plugin
+    # was not used.)
+    if (session.config.getoption("keep_workflow_wd") or
+            len(session.config.workflow_cleanup_dirs) == 0):
+        pass
+    elif session.config.getoption("keep_workflow_wd_on_fail"):
+        # Ony cleanup if there are cleanup_dirs.
+        if exitstatus == 0:
+            print("All tests succeeded. Removing temporary directories and "
+                  "logs.")
+            rm_dirs(session.config.workflow_cleanup_dirs)
+        else:
+            print("One or more tests failed. Keeping temporary directories "
+                  "and logs.")
+    else:  # When no flags are set. Remove temporary directories and logs.
+        print("Removing temporary directories and logs. Use '--kwd' or "
               "'--keep-workflow-wd' to disable this behaviour.")
-        for tempdir in session.config.workflow_cleanup_dirs:
-            shutil.rmtree(str(tempdir))
+        rm_dirs(session.config.workflow_cleanup_dirs)
 
 
 @pytest.fixture()

tests/test_temp_directory.py

@@ -25,7 +25,7 @@
 def test_directory_kept(testdir):
     testdir.makefile(".yml", test=SIMPLE_ECHO)
     result = testdir.runpytest("-v", "--keep-workflow-wd")
-    working_dir = re.search(r"with command 'echo moo' in '([\w\/_-]*)'",
+    working_dir = re.search(r"with command 'echo moo' in '([\w/_-]*)'",
                             result.stdout.str()).group(1)
     assert Path(working_dir).exists()
     assert Path(working_dir / Path("log.out")).exists()
@@ -35,7 +35,7 @@ def test_directory_kept(testdir):
 def test_directory_not_kept(testdir):
     testdir.makefile(".yml", test=SIMPLE_ECHO)
     result = testdir.runpytest("-v")
-    working_dir = re.search(r"with command 'echo moo' in '([\w\/_-]*)'",
+    working_dir = re.search(r"with command 'echo moo' in '([\w/_-]*)'",
                             result.stdout.str()).group(1)
     assert not Path(working_dir).exists()
     assert ("Removing temporary directories and logs. Use '--kwd' or "
@@ -83,3 +83,49 @@ def test_basetemp_will_be_created(testdir):
                                str(tempdir))
     assert tempdir.exists()
     assert result.ret == 0
+
+
+def test_basetemp_can_not_be_in_rootdir(testdir):
+    testdir.makefile(".yml", test=SIMPLE_ECHO)
+    testdir_path = Path(str(testdir.tmpdir))
+    tempdir = testdir_path / Path("tmp")
+    result = testdir.runpytest("-v", "--basetemp", str(tempdir))
+    message = "'{tempdir}' is a subdirectory of '{rootdir}'".format(
+        tempdir=str(tempdir),
+        rootdir=str(testdir_path))
+    assert message in result.stderr.str()
+
+
+SUCCESS_TEST = """\
+- name: success
+  command: bash -c 'exit 0'
+"""
+
+FAIL_TEST = """\
+- name: fail
+  command: bash -c 'exit 1'
+"""
+
+
+def test_directory_kept_on_fail(testdir):
+    testdir.makefile(".yml", test=FAIL_TEST)
+    result = testdir.runpytest("-v", "--keep-workflow-wd-on-fail")
+    working_dir = re.search(
+        r"with command 'bash -c 'exit 1'' in '([\w/_-]*)'",
+        result.stdout.str()).group(1)
+    assert Path(working_dir).exists()
+    assert Path(working_dir / Path("log.out")).exists()
+    assert Path(working_dir / Path("log.err")).exists()
+    assert ("One or more tests failed. Keeping temporary directories and "
+            "logs." in result.stdout.str())
+
+
+def test_directory_not_kept_on_succes(testdir):
+    testdir.makefile(".yml", test=SUCCESS_TEST)
+    result = testdir.runpytest("-v", "--kwdof")
+    working_dir = re.search(
+        r"with command 'bash -c 'exit 0'' in '([\w/_-]*)'",
+        result.stdout.str()).group(1)
+    assert not Path(working_dir).exists()
+    assert ("All tests succeeded. Removing temporary directories and logs." in
+            result.stdout.str())