From c5088d3da0eebdaec97de89bb1063d3c4c37b3cb Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Fri, 19 Apr 2024 10:07:35 -0400 Subject: [PATCH 1/7] fix github action for publishing --- .github/workflows/python-publish.yml | 35 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 48c52e1..a910e5a 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -9,22 +9,23 @@ on: jobs: deploy: + name: upload release to PyPI runs-on: ubuntu-latest - + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + run: | + python setup.py sdist bdist_wheel + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + From 7887306998198ca7471953a590ae5d127792436b Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Sun, 5 May 2024 02:29:20 +0200 Subject: [PATCH 2/7] remove unused line that's now accomplished in subroutine --- pypiper/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pypiper/manager.py b/pypiper/manager.py index 965945b..da12953 100644 --- a/pypiper/manager.py +++ b/pypiper/manager.py @@ -1290,7 +1290,6 @@ def proc_wrapup(i): sleeptime = min((sleeptime + 0.25) * 3, 60 / len(processes)) # All jobs are done, print a final closing and job info - stop_time = time.time() proc_message = "Command completed. {info}" info = ( "Elapsed time: " From 07fb60f474c4e3e330172ad53bfc6e21e1346bb6 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Sun, 5 May 2024 02:30:01 +0200 Subject: [PATCH 3/7] remove only-once-used variable binding --- pypiper/manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pypiper/manager.py b/pypiper/manager.py index da12953..657dd39 100644 --- a/pypiper/manager.py +++ b/pypiper/manager.py @@ -1290,7 +1290,6 @@ def proc_wrapup(i): sleeptime = min((sleeptime + 0.25) * 3, 60 / len(processes)) # All jobs are done, print a final closing and job info - proc_message = "Command completed. {info}" info = ( "Elapsed time: " + str(datetime.timedelta(seconds=self.time_elapsed(start_time))) @@ -1307,7 +1306,7 @@ def proc_wrapup(i): info += "\n" # finish out the self.info("") - self.info(proc_message.format(info=info)) + self.info("Command completed. {info}".format(info=info)) for rc in returncodes: if rc != 0: From 90fb7ad532b3f954fc649fd99ad33dc318a01f56 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Sun, 5 May 2024 02:32:46 +0200 Subject: [PATCH 4/7] allow Stage to be nofail, close #218 --- pypiper/pipeline.py | 5 ++++- pypiper/stage.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pypiper/pipeline.py b/pypiper/pipeline.py index 95d31f2..ab3f8d9 100644 --- a/pypiper/pipeline.py +++ b/pypiper/pipeline.py @@ -332,7 +332,10 @@ def run(self, start_point=None, stop_before=None, stop_after=None): print(f"Running stage: {getattr(stage, 'name', str(stage))}") - stage.run() + try: + stage.run() + except Exception as e: + self.manager._triage_error(e, nofail=stage.nofail) self.executed.append(stage) self.checkpoint(stage) diff --git a/pypiper/stage.py b/pypiper/stage.py index 6f1d551..929becf 100644 --- a/pypiper/stage.py +++ b/pypiper/stage.py @@ -17,7 +17,16 @@ class Stage(object): collection of commands that is checkpointed. """ - def __init__(self, func, f_args=None, f_kwargs=None, name=None, checkpoint=True): + def __init__( + self, + func, + f_args=None, + f_kwargs=None, + name=None, + checkpoint=True, + *, + nofail=False + ): """ A function, perhaps with arguments, defines the stage. @@ -26,6 +35,8 @@ def __init__(self, func, f_args=None, f_kwargs=None, name=None, checkpoint=True) :param dict f_kwargs: Keyword arguments for func :param str name: name for the phase/stage :param callable func: Object that defines how the stage will execute. + :param bool nofail: Allow a failure of this stage to not fail the pipeline + in which it's running """ if isinstance(func, Stage): raise TypeError("Cannot create Stage from Stage") @@ -35,6 +46,7 @@ def __init__(self, func, f_args=None, f_kwargs=None, name=None, checkpoint=True) self.f_kwargs = f_kwargs or dict() self.name = name or func.__name__ self.checkpoint = checkpoint + self.nofail = nofail @property def checkpoint_name(self): From 7fa5a67181b589f0e67c68090dc15b72ae7d39c2 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Sun, 5 May 2024 02:39:48 +0200 Subject: [PATCH 5/7] add a requirements file tracking extra dev dependencies --- requirements/requirements-dev-extra.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements/requirements-dev-extra.txt diff --git a/requirements/requirements-dev-extra.txt b/requirements/requirements-dev-extra.txt new file mode 100644 index 0000000..7e66a17 --- /dev/null +++ b/requirements/requirements-dev-extra.txt @@ -0,0 +1 @@ +black From 0edf2f2b9402ab1b028efddcae71a9db275aacd1 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Tue, 7 May 2024 02:23:29 +0200 Subject: [PATCH 6/7] don't checkpoint a nofail stage when it's nofailed --- pypiper/pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pypiper/pipeline.py b/pypiper/pipeline.py index ab3f8d9..a02c9ad 100644 --- a/pypiper/pipeline.py +++ b/pypiper/pipeline.py @@ -336,8 +336,9 @@ def run(self, start_point=None, stop_before=None, stop_after=None): stage.run() except Exception as e: self.manager._triage_error(e, nofail=stage.nofail) + else: + self.checkpoint(stage) self.executed.append(stage) - self.checkpoint(stage) # Add any unused stages to the collection of skips. self.skipped.extend(self._stages[stop_index:]) From 59130c398da67bcabf3fbd8a14e50e99c30adac4 Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Tue, 7 May 2024 11:56:21 -0400 Subject: [PATCH 7/7] update version and changelog in preparation for 0.14.2 --- docs/changelog.md | 4 ++++ pypiper/_version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index feba103..35fec73 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,9 @@ # Changelog +## [0.14.2] -- 2024-05-07 +### Changed +- Addresses [#218](https://github.com/databio/pypiper/issues/218) + ## [0.14.1] -- 2024-04-19 ### Changed - remove pipestat_project_name from PipelineManager parameters diff --git a/pypiper/_version.py b/pypiper/_version.py index f075dd3..745162e 100644 --- a/pypiper/_version.py +++ b/pypiper/_version.py @@ -1 +1 @@ -__version__ = "0.14.1" +__version__ = "0.14.2"